From e9644510fd36601e641e0084db84cf433a7df753 Mon Sep 17 00:00:00 2001 From: litaotao Date: Mon, 10 Oct 2016 15:15:39 +0800 Subject: [PATCH 1/4] update client structure --- .gitignore | 4 + mercury/__init__.py | 2 +- mercury/data.py | 228 -------------------------------------------- mercury/uqer.py | 99 +++++++++++++++++++ mercury/utils.py | 125 ++++++++++++++++++++++++ 5 files changed, 229 insertions(+), 229 deletions(-) create mode 100644 .gitignore delete mode 100644 mercury/data.py create mode 100644 mercury/uqer.py create mode 100644 mercury/utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..494bf37 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.ipynb_checkpoints/* +*.ipynb +*.txt +*.pyc diff --git a/mercury/__init__.py b/mercury/__init__.py index 3c4096e..c1f64a1 100644 --- a/mercury/__init__.py +++ b/mercury/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -from data import Client \ No newline at end of file +from uqer import Client \ No newline at end of file diff --git a/mercury/data.py b/mercury/data.py deleted file mode 100644 index 7890114..0000000 --- a/mercury/data.py +++ /dev/null @@ -1,228 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - data.py - ~~~~~~~~~~~~ - - Download data from DataYes Mercury. - - :copyright: (c) 2015 by DataYes Fixed Income Team. - :Author: taotao.li - :last updated: Mar.10th.2014 -""" - -import sys -import os -import ConfigParser -import requests - - -DEBUG = False -AUTHORIZE_URL = "https://gw.wmcloud.com/usermaster/authenticate.json" -MERCURY_URL = 'https://gw.wmcloud.com/mercury/api/databooks' -DOWNLOAD_URL = 'https://gw.wmcloud.com/mercury/databooks' -NOTEBOOK_URL = 'https://gw.wmcloud.com/mercury/api/notebooks' -DOWNLOAD_NOTEBOOK_URL = 'https://gw.wmcloud.com/mercury/files' - - -class Client(object): - """DataYes Mercury Client - - Methods for the caller: - - - __init__(username, password) - username and password here are which used by users to login www.datayes.com - - lists() - Show the all the data in one user's mercury data zone. - - get(filename='', download_all=False) - Get user's data according to filename, can be a string or a list of string. - If set all to True, will download all the data file. - - delete(filename) - Delete user's data according to filename, can only be a string. - """ - def __init__(self, username, password, token=''): - if DEBUG: - import pdb; pdb.set_trace() - if not token: - self.username = username - self.password = password - print 'Welcome, {} ... '.format(username) - self.isvalid, self.token = authorize_user(username, password) - self.cookies = {'cloud-sso-token': self.token} - if not self.isvalid: - print 'Sorry, {}, your username or password are not match, authorization failed ...'.format(username) - else: - self.isvalid = True - self.cookies = {'cloud-sso-token': token} - - def lists(self): - ''' - Show the all the data in one user's mercury data zone. - ''' - if not self.isvalid: - print 'Sorry, {}, your username or password are not match, authorization failed ...' - reutrn - self.all_data = list_data(self.cookies) - self.all_notebook = list_notebook(self.cookies) - - - def get(self, filename='', download_all=False): - ''' - Get user's data according to filename, can be a string or a list of string. - If set all to True, will download all the data file. - ''' - if not self.isvalid: - print 'Sorry, {}, your username or password are not match, authorization failed ...' - reutrn - - if download_all: - self.lists() - for i in self.all_data: - download_file(self.cookies, i) - return True - elif type(filename) == list: - for i in filename: - download_file(self.cookies, i) - return True - elif type(filename) == str: - download_file(self.cookies, filename) - return True - else: - pass - - return False - - def notebook(self, filename='', download_all=True): - ''' - Get user's notebook according to filename, can be a string or a list of string. - If set all to True, will download all the notebook file, just for back up. - ''' - if not self.isvalid: - print 'Sorry, {}, your username or password are not match, authorization failed ...' - reutrn - - if download_all: - self.lists() - for i in self.all_notebook: - download_notebook(self.cookies, i) - return True - elif type(filename) == list: - for i in filename: - download_notebook(self.cookies, i) - return True - elif type(filename) == str: - download_notebook(self.cookies, filename) - return True - else: - pass - - return False - - def push(self, filepath): - ''' - Push a file to your DataYes Mercury zone. - ''' - try: - files = {'datafile': open(filepath, 'rb')} - except: - print u"Can not open file at: ".format(filepath) - return False - - r = requests.post(MERCURY_URL, files=files, cookies=self.cookies) - - print r.json().get('message', '') if not r.ok else '' - - return r.ok - - def delete(self, filename): - ''' - Delete user's data according to filename, can only be a string. - ''' - res = delete_file(self.cookies, filename) - if res: - print u'Delete file {} done ...'.format(filename) - else: - print u'Something is wrong when trying to delete file {} ...'.format(filename) - - -def authorize_user(user, pwd): - url = AUTHORIZE_URL - if '@' in user: - user, tenant = user.split("@") - else: - return False, None - data = dict(username=user, password=pwd, tenant=tenant) - res = requests.post(url, data) - if not res.ok or not res.json().get('content', {}).get('accountId', 0): - return False, None - else: - token = res.json().get('content', {}).get('token', {}).get('tokenString', '') - return True, token - -def list_data(cookies): - url = MERCURY_URL - res = requests.get(url, cookies=cookies) - if not res.ok: - print 'Request error, maybe a server error, please retry or contact us directly' - return 0 - data = res.json() - print "Hello, there are {} files in your DataYes Mercury VM".format(str(len(data))) - all_data = [i['name'] for i in data] - for i in all_data: - print u'Name: {}'.format(i) - - return all_data - -def list_notebook(cookies): - url = NOTEBOOK_URL - res = requests.get(url, cookies=cookies) - if not res.ok: - print 'Request error, maybe a server error, please retry or contact us directly' - return 0 - data = res.json() - print "Hello, there are {} notebooks in your DataYes Mercury VM".format(str(len(data))) - all_notebook = [i['name'] for i in data] - for i in all_notebook: - print u'Name: {}'.format(i) - - return all_notebook - -def download_notebook(cookies, filename): - url = DOWNLOAD_NOTEBOOK_URL - notebook_url = url + '/' + filename - print u'\nStart download {}'.format(filename), - - with open(filename, 'wb') as f: - response = requests.get(notebook_url, cookies=cookies, stream=True) - - if not response.ok: - print u'Something is wrong when download file {} '.format(filename) - return 0 - - for chunk in response.iter_content(1024 * 100): - print '...', - f.write(chunk) - -def download_file(cookies, filename): - url = DOWNLOAD_URL - dataurl = url + '/' + filename - print u'\nStart download {}'.format(filename), - - with open(filename, 'wb') as f: - response = requests.get(dataurl, cookies=cookies, stream=True) - - if not response.ok: - print u'Something is wrong when download file {} '.format(filename) - return 0 - - for chunk in response.iter_content(1024 * 100): - print '...', - f.write(chunk) - -def delete_file(cookies, filename): - url = MERCURY_URL - deleteurl = url + '/' + filename - res = requests.delete(deleteurl, cookies=cookies) - - return res.ok - \ No newline at end of file diff --git a/mercury/uqer.py b/mercury/uqer.py new file mode 100644 index 0000000..45bea7a --- /dev/null +++ b/mercury/uqer.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- + +""" + uqer.py + ~~~~~~~~~~~~ + + + :copyright: (c) 2016 by DataYes Fixed Income Team. + :Author: taotao.li +""" + +import sys +import os +import ConfigParser +import requests + + +import utils + + +class Client(object): + """优矿 + """ + def __init__(self, username='', password='', token=''): + if not token: + self.username = username + self.password = password + print 'Welcome, {} ... '.format(username) + self.isvalid, self.token = utils.authorize_user(username, password) + self.cookies = {'cloud-sso-token': self.token} + if not self.isvalid: + print 'Sorry, {}, your username or password are not match, authorization failed ...'.format(username) + else: + self.isvalid = True + self.cookies = {'cloud-sso-token': token} + + def list_data(self): + if not self.isvalid: + print 'Sorry, {}, your username or password are not match, authorization failed ...' + return + + self.all_data = utils.list_data(self.cookies) + + def list_notebook(self): + if not self.isvalid: + print 'Sorry, {}, your username or password are not match, authorization failed ...' + return + + self.all_notebook = utils.list_notebook(self.cookies) + + def download_data(self, filename='', download_all=False): + if not self.isvalid: + print 'Sorry, {}, your username or password are not match, authorization failed ...' + return + + if download_all: + self.list_data() + for i in self.all_data: + utils.download_file(self.cookies, i) + + elif type(filename) == list: + for i in filename: + utils.download_file(self.cookies, i) + + elif type(filename) == str: + utils.download_file(self.cookies, filename) + + else: + pass + + def download_notebook(self, filename='', download_all=False): + if not self.isvalid: + print 'Sorry, {}, your username or password are not match, authorization failed ...' + return + + if download_all: + self.list_notebook() + for i in self.all_notebook: + utils.download_notebook(self.cookies, i) + + elif type(filename) == list: + for i in filename: + utils.download_notebook(self.cookies, i) + + elif type(filename) in (str, unicode): + utils.download_notebook(self.cookies, filename) + + else: + pass + + # def upload_data(self, filepath): + # import ipdb; ipdb.set_trace() + # try: + # files = {'datafile': open(filepath, 'rb')} + # except: + # print u"Can not open file at: ".format(filepath) + # return False + + # utils.upload_data(files, self.cookies) diff --git a/mercury/utils.py b/mercury/utils.py new file mode 100644 index 0000000..3ae2a6b --- /dev/null +++ b/mercury/utils.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- + +""" + utils.py + ~~~~~~~~~~~~ + + + :copyright: (c) 2016 by DataYes Fixed Income Team. + :Author: taotao.li +""" + +import sys +import os +import ConfigParser +import urllib +import requests + + +AUTHORIZE_URL = "https://gw.wmcloud.com/usermaster/authenticate.json" +MERCURY_URL = 'https://gw.wmcloud.com/mercury/api/databooks' +DOWNLOAD_DATA_URL = 'https://gw.wmcloud.com/mercury/databooks' +NOTEBOOK_URL = 'https://gw.wmcloud.com/mercury/api/notebooks?recursion' +DOWN_NOTEBOOK_URL = 'https://gw.wmcloud.com/mercury/files' +FOLDERS = [] +LOCAL_PATH = './' + + +def authorize_user(user, pwd): + url = AUTHORIZE_URL + if '@' in user: + user, tenant = user.split("@") + else: + return False, None + data = dict(username=user, password=pwd, tenant=tenant) + res = requests.post(url, data) + if not res.ok or not res.json().get('content', {}).get('accountId', 0): + return False, None + else: + token = res.json().get('content', {}).get('token', {}).get('tokenString', '') + return True, token + +def list_data(cookies): + url = MERCURY_URL + res = requests.get(url, cookies=cookies) + if not res.ok: + print 'Request error, maybe a server error, please retry or contact us directly' + return 0 + + data = res.json() + print "Hello, there are {} files in your DataYes Mercury VM".format(str(len(data))) + all_data = [i['name'] for i in data] + for i in all_data: + print u'Name: {}'.format(i) + + return all_data + +def list_notebook(cookies): + global FOLDERS + url = NOTEBOOK_URL + res = requests.get(url, cookies=cookies) + if not res.ok: + print 'Request error, maybe a server error, please retry or contact us directly' + return 0 + data = res.json() + all_notebook = [] + for i in data: + if i['type'] == 'directory': + FOLDERS.append(i['name']) + for j in i['children']: + all_notebook.append(u'{}/{}'.format(i['name'], j['name'])) + elif i['type'] == 'notebook': + all_notebook.append(u'{}'.format(i['name'])) + else: + pass + print "Hello, there are {} notebooks in your DataYes Mercury VM".format(str(len(all_notebook))) + for i in all_notebook: + print u'Name: {}'.format(i) + + return all_notebook + +def download_notebook(cookies, filename): + global FOLDERS + url = DOWN_NOTEBOOK_URL + folders = set(FOLDERS) + notebook_url = url + '/' + urllib.quote(filename.encode('utf-8')) + print u'\nStart download {}'.format(filename), + print notebook_url + + filename = filename.split('/')[-1] + with open(LOCAL_PATH + filename, 'wb') as f: + response = requests.get(notebook_url, cookies=cookies, stream=True) + + if not response.ok: + print u'Something is wrong when download file {} '.format(filename) + return 0 + + for chunk in response.iter_content(1024 * 100): + print '...', + f.write(chunk) + +def download_file(cookies, filename): + url = DOWNLOAD_DATA_URL + dataurl = url + '/' + filename + print '\nStart download {}'.format(filename), + + with open(filename, 'wb') as f: + response = requests.get(dataurl, cookies=cookies, stream=True) + + if not response.ok: + print u'Something is wrong when download file {} '.format(filename) + return 0 + + for chunk in response.iter_content(1024 * 100): + print '...', + f.write(chunk) + + +def upload_data(files, cookies): + headers = {'Content-Type': 'multipart/form-data'} + r = requests.post(MERCURY_URL, data=files, cookies=cookies, headers=headers) + print r.text + + print r.json().get('message', '') if not r.ok else '' + + return r.ok \ No newline at end of file From 87336b9825c82342eb9918f1f3317b4842786692 Mon Sep 17 00:00:00 2001 From: litaotao Date: Tue, 11 Oct 2016 08:31:19 +0800 Subject: [PATCH 2/4] add backup shortcut --- README.md | 57 ++---------------------------------------------- mercury/uqer.py | 6 +++++ mercury/utils.py | 1 - 3 files changed, 8 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 2be79ca..5be9c13 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,5 @@ -# mercury-client - - -# Tutorial - - -## 1. Install python 2.7 - -## 2. Install mercury-client package from Github - - Datayes Official Github is: https://github.com/DataYes/mercury-client, You can download the mercury-client package here. - Or download directly from [here](http://litaotao.github.io/files/mercuryclient.rar) - -### 2.1 Uncompress the package -### 2.2 Intall the package - -- Open the command line tool - - windows: cmd - - linux/mac: terminal -- Use the ***cd*** command to go the the package directory which contains ***setup.py*** file -- Execute command ***python setup.py install*** to install the mercury-client package - -## 3. USAGE: - -***Steps*** - -- Get an client instance - - import mercury - - client = mercury.Client(username='username', password='password') -- Use that instance to list/get/delete your files in Datayes Mercury. - - lists() - Show the all the data in one user's mercury data zone. - - get(filename='', download_all=False) - Get user's data according to filename, can be a string or a list of string. - If set all to True, will download all the data file. - - delete(filename) - Delete user's data according to filename, can only be a string. - -***EXAMPLES*** - - import mercury - client = mercury.Client('taotao.li@datayes.com', 'password') - all_files = client.lists() - client.get(filename='123.txt') - client.delete(filename='123.txt') - -Bellow is the screenshot of the above: -![mercury-client.jpg](http://litaotao.github.io/images/mercury-client.jpg) - -# 中文使用步骤 - -- 安装Python 2.7 -- 安装mercury-client包 -- 生成client实例 -- 使用client进行list/get/delete操作 +## Uqer Client +使用步骤：参考优矿社区贴： diff --git a/mercury/uqer.py b/mercury/uqer.py index 45bea7a..a64f005 100644 --- a/mercury/uqer.py +++ b/mercury/uqer.py @@ -88,6 +88,12 @@ def download_notebook(self, filename='', download_all=False): else: pass + def backup_data(self): + self.download_data(download_all=True) + + def backup_notebook(self): + self.download_data(download_all=True) + # def upload_data(self, filepath): # import ipdb; ipdb.set_trace() # try: diff --git a/mercury/utils.py b/mercury/utils.py index 3ae2a6b..b74d287 100644 --- a/mercury/utils.py +++ b/mercury/utils.py @@ -114,7 +114,6 @@ def download_file(cookies, filename): print '...', f.write(chunk) - def upload_data(files, cookies): headers = {'Content-Type': 'multipart/form-data'} r = requests.post(MERCURY_URL, data=files, cookies=cookies, headers=headers) From 90c333e3d24a0643d498d6612077a6cab269bb6c Mon Sep 17 00:00:00 2001 From: litaotao Date: Tue, 11 Oct 2016 11:01:12 +0800 Subject: [PATCH 3/4] test done --- .gitignore | 4 ++++ mercury/uqer.py | 42 ++++++++++++++++++++---------------------- mercury/utils.py | 10 ++++------ 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 494bf37..d9ce055 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ *.ipynb *.txt *.pyc +*.csv +*.xls* +daily_report +manual_adjust \ No newline at end of file diff --git a/mercury/uqer.py b/mercury/uqer.py index a64f005..aaa7b6c 100644 --- a/mercury/uqer.py +++ b/mercury/uqer.py @@ -23,30 +23,28 @@ class Client(object): """ def __init__(self, username='', password='', token=''): if not token: - self.username = username - self.password = password print 'Welcome, {} ... '.format(username) self.isvalid, self.token = utils.authorize_user(username, password) - self.cookies = {'cloud-sso-token': self.token} + self.__cookies = {'cloud-sso-token': self.token} if not self.isvalid: print 'Sorry, {}, your username or password are not match, authorization failed ...'.format(username) else: self.isvalid = True - self.cookies = {'cloud-sso-token': token} + self.__cookies = {'cloud-sso-token': token} def list_data(self): if not self.isvalid: print 'Sorry, {}, your username or password are not match, authorization failed ...' return - self.all_data = utils.list_data(self.cookies) + self.__all_data = utils.list_data(self.__cookies) def list_notebook(self): if not self.isvalid: print 'Sorry, {}, your username or password are not match, authorization failed ...' return - self.all_notebook = utils.list_notebook(self.cookies) + self.__all_notebook = utils.list_notebook(self.__cookies) def download_data(self, filename='', download_all=False): if not self.isvalid: @@ -55,15 +53,15 @@ def download_data(self, filename='', download_all=False): if download_all: self.list_data() - for i in self.all_data: - utils.download_file(self.cookies, i) + for i in self.__all_data: + utils.download_file(self.__cookies, i) elif type(filename) == list: for i in filename: - utils.download_file(self.cookies, i) + utils.download_file(self.__cookies, i) elif type(filename) == str: - utils.download_file(self.cookies, filename) + utils.download_file(self.__cookies, filename) else: pass @@ -75,15 +73,15 @@ def download_notebook(self, filename='', download_all=False): if download_all: self.list_notebook() - for i in self.all_notebook: - utils.download_notebook(self.cookies, i) + for i in self.__all_notebook: + utils.download_notebook(self.__cookies, i) elif type(filename) == list: for i in filename: - utils.download_notebook(self.cookies, i) + utils.download_notebook(self.__cookies, i) elif type(filename) in (str, unicode): - utils.download_notebook(self.cookies, filename) + utils.download_notebook(self.__cookies, filename) else: pass @@ -94,12 +92,12 @@ def backup_data(self): def backup_notebook(self): self.download_data(download_all=True) - # def upload_data(self, filepath): - # import ipdb; ipdb.set_trace() - # try: - # files = {'datafile': open(filepath, 'rb')} - # except: - # print u"Can not open file at: ".format(filepath) - # return False + def upload_data(self, filepath): + try: + f = open(filepath, 'rb') + files = {'datafile': f} + except: + print u"Can not open file at: ".format(filepath) + return False - # utils.upload_data(files, self.cookies) + utils.upload_data(files, self.__cookies) diff --git a/mercury/utils.py b/mercury/utils.py index b74d287..afa7a48 100644 --- a/mercury/utils.py +++ b/mercury/utils.py @@ -97,6 +97,8 @@ def download_notebook(cookies, filename): for chunk in response.iter_content(1024 * 100): print '...', f.write(chunk) + print u'\nDown download {}'.format(filename) + def download_file(cookies, filename): url = DOWNLOAD_DATA_URL @@ -113,12 +115,8 @@ def download_file(cookies, filename): for chunk in response.iter_content(1024 * 100): print '...', f.write(chunk) + print '\nDown download {}'.format(filename) def upload_data(files, cookies): - headers = {'Content-Type': 'multipart/form-data'} - r = requests.post(MERCURY_URL, data=files, cookies=cookies, headers=headers) + r = requests.post(MERCURY_URL, files=files, cookies=cookies) print r.text - - print r.json().get('message', '') if not r.ok else '' - - return r.ok \ No newline at end of file From 29d37ddc7c3e2dd16d66272450b7a4594d19fca0 Mon Sep 17 00:00:00 2001 From: litaotao Date: Tue, 11 Oct 2016 11:03:39 +0800 Subject: [PATCH 4/4] test all done --- mercury/uqer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mercury/uqer.py b/mercury/uqer.py index aaa7b6c..e5564ea 100644 --- a/mercury/uqer.py +++ b/mercury/uqer.py @@ -90,7 +90,7 @@ def backup_data(self): self.download_data(download_all=True) def backup_notebook(self): - self.download_data(download_all=True) + self.download_notebook(download_all=True) def upload_data(self, filepath): try: