Merge pull request #118 from qiniu/develop

Release 6.1.6

Author: longbai

CHANGELOG.md

@@ -1,5 +1,14 @@
 ## CHANGE LOG
 
+### v6.1.6
+
+2014-06-30 issue [#118](https://github.com/qiniu/python-sdk/pull/118)
+- [#108] 增加限定用户上传类型字段
+- [#110] 0字节文件上传支持
+- [#114] 支持GAE
+- [#115] 请求完成后清除http header
+- [#120] 增加第二个up host重试
+
 ### v6.1.5
 
 2014-04-28 issue [#105](https://github.com/qiniu/python-sdk/pull/105)

qiniu/auth/digest.py

@@ -55,8 +55,8 @@ def __init__(self, host, mac=None):
         super(Client, self).__init__(host)
         self.mac = mac
 
-    def round_tripper(self, method, path, body):
+    def round_tripper(self, method, path, body, header={}):
         token = self.mac.sign_request(
-            path, body, self._header.get("Content-Type"))
-        self.set_header("Authorization", "QBox %s" % token)
-        return super(Client, self).round_tripper(method, path, body)
+            path, body, header.get("Content-Type"))
+        header["Authorization"] = "QBox %s" % token
+        return super(Client, self).round_tripper(method, path, body, header)

qiniu/auth/up.py

@@ -14,6 +14,6 @@ def __init__(self, up_token, host=None):
         self.up_token = up_token
         super(Client, self).__init__(host)
 
-    def round_tripper(self, method, path, body):
-        self.set_header("Authorization", "UpToken %s" % self.up_token)
-        return super(Client, self).round_tripper(method, path, body)
+    def round_tripper(self, method, path, body, header={}):
+        header["Authorization"] = "UpToken %s" % self.up_token
+        return super(Client, self).round_tripper(method, path, body, header)

qiniu/conf.py

@@ -6,6 +6,7 @@
 RS_HOST = "rs.qbox.me"
 RSF_HOST = "rsf.qbox.me"
 UP_HOST = "up.qiniu.com"
+UP_HOST2 = "up.qbox.me"
 
 from . import __version__
 import platform

qiniu/io.py

@@ -52,7 +52,12 @@ def put(uptoken, key, data, extra=None):
     files = [
         {'filename': fname, 'data': data, 'mime_type': extra.mime_type},
     ]
-    return rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files)
+    ret, err, code = rpc.Client(conf.UP_HOST).call_with_multipart("/", fields, files)
+    if err is None or code == 571 or code == 614 or code == 301:
+        return ret, err
+
+    ret, err, code = rpc.Client(conf.UP_HOST2).call_with_multipart("/", fields, files)
+    return ret, err
 
 
 def put_file(uptoken, key, localfile, extra=None):

qiniu/resumable_io.py

@@ -66,23 +66,35 @@ def put_file(uptoken, key, localfile, extra):
     """ 上传文件 """
     f = open(localfile, "rb")
     statinfo = os.stat(localfile)
-    ret = put(uptoken, key, f, statinfo.st_size, extra)
+    ret, err = put(uptoken, key, f, statinfo.st_size, extra)
     f.close()
-    return ret
+    return ret, err
 
 
 def put(uptoken, key, f, fsize, extra):
     """ 上传二进制流, 通过将data "切片" 分段上传 """
     if not isinstance(extra, PutExtra):
         print("extra must the instance of PutExtra")
         return
+    host = conf.UP_HOST
+    try:
+        ret, err, code = put_with_host(uptoken, key, f, fsize, extra, host)
+        if err is None or code == 571 or code == 614 or code == 301:
+            return ret, err
+    except:
+        pass
 
+    ret, err, code = put_with_host(uptoken, key, f, fsize, extra, conf.UP_HOST2)
+    return ret, err
+
+
+def put_with_host(uptoken, key, f, fsize, extra, host):
     block_cnt = block_count(fsize)
     if extra.progresses is None:
         extra.progresses = [None] * block_cnt
     else:
         if not len(extra.progresses) == block_cnt:
-            return None, err_invalid_put_progress
+            return None, err_invalid_put_progress, 0
 
     if extra.try_times is None:
         extra.try_times = _try_times
@@ -97,27 +109,29 @@ def put(uptoken, key, f, fsize, extra):
             read_length = fsize - i * _block_size
         data_slice = f.read(read_length)
         while True:
-            err = resumable_block_put(data_slice, i, extra, uptoken)
+            err = resumable_block_put(data_slice, i, extra, uptoken, host)
             if err is None:
                 break
 
             try_time -= 1
             if try_time <= 0:
-                return None, err_put_failed
+                return None, err_put_failed, 0
             print err, ".. retry"
 
-    mkfile_client = auth_up.Client(uptoken, extra.progresses[-1]["host"])
-    return mkfile(mkfile_client, key, fsize, extra)
+    mkfile_host = extra.progresses[-1]["host"] if block_cnt else host
+    mkfile_client = auth_up.Client(uptoken, mkfile_host)
+
+    return mkfile(mkfile_client, key, fsize, extra, host)
 
 
-def resumable_block_put(block, index, extra, uptoken):
+def resumable_block_put(block, index, extra, uptoken, host):
     block_size = len(block)
 
-    mkblk_client = auth_up.Client(uptoken, conf.UP_HOST)
+    mkblk_client = auth_up.Client(uptoken, host)
     if extra.progresses[index] is None or "ctx" not in extra.progresses[index]:
         crc32 = gen_crc32(block)
         block = bytearray(block)
-        extra.progresses[index], err = mkblock(mkblk_client, block_size, block)
+        extra.progresses[index], err, code = mkblock(mkblk_client, block_size, block, host)
         if err is not None:
             extra.notify_err(index, block_size, err)
             return err
@@ -132,8 +146,8 @@ def block_count(size):
     return (size + _block_mask) / _block_size
 
 
-def mkblock(client, block_size, first_chunk):
-    url = "http://%s/mkblk/%s" % (conf.UP_HOST, block_size)
+def mkblock(client, block_size, first_chunk, host):
+    url = "http://%s/mkblk/%s" % (host, block_size)
     content_type = "application/octet-stream"
     return client.call_with(url, first_chunk, content_type, len(first_chunk))
 
@@ -145,8 +159,8 @@ def putblock(client, block_ret, chunk):
     return client.call_with(url, chunk, content_type, len(chunk))
 
 
-def mkfile(client, key, fsize, extra):
-    url = ["http://%s/mkfile/%s" % (conf.UP_HOST, fsize)]
+def mkfile(client, key, fsize, extra, host):
+    url = ["http://%s/mkfile/%s" % (host, fsize)]
 
     if extra.mimetype:
         url.append("mimeType/%s" % urlsafe_b64encode(extra.mimetype))

qiniu/rpc.py

@@ -1,5 +1,11 @@
 # -*- coding: utf-8 -*-
-import httplib_chunk as httplib
+
+import httplib
+
+if getattr(httplib, "_IMPLEMENTATION", None) != "gae":
+    # httplib._IMPLEMENTATION is "gae" on GAE
+    import httplib_chunk as httplib
+
 import json
 import cStringIO
 import conf
@@ -13,25 +19,32 @@ def __init__(self, host):
         self._conn = httplib.HTTPConnection(host)
         self._header = {}
 
-    def round_tripper(self, method, path, body):
-        self._conn.request(method, path, body, self._header)
+    def round_tripper(self, method, path, body, header={}):
+        header = self.merged_headers(header)
+        self._conn.request(method, path, body, header)
         resp = self._conn.getresponse()
         return resp
 
+    def merged_headers(self, header):
+        _header = self._header.copy()
+        _header.update(header)
+        return _header
+
     def call(self, path):
-        return self.call_with(path, None)
+        ret, err, code = self.call_with(path, None)
+        return ret, err
 
     def call_with(self, path, body, content_type=None, content_length=None):
         ret = None
 
-        self.set_header("User-Agent", conf.USER_AGENT)
+        header = {"User-Agent": conf.USER_AGENT}
         if content_type is not None:
-            self.set_header("Content-Type", content_type)
+            header["Content-Type"] = content_type
 
         if content_length is not None:
-            self.set_header("Content-Length", content_length)
+            header["Content-Length"] = content_length
 
-        resp = self.round_tripper("POST", path, body)
+        resp = self.round_tripper("POST", path, body, header)
         try:
             ret = resp.read()
             ret = json.loads(ret)
@@ -40,16 +53,16 @@ def call_with(self, path, body, content_type=None, content_length=None):
         except ValueError:
             pass
 
-        if resp.status / 100 != 2:
+        if resp.status >= 400:
             err_msg = ret if "error" not in ret else ret["error"]
             reqid = resp.getheader("X-Reqid", None)
             # detail = resp.getheader("x-log", None)
             if reqid is not None:
                 err_msg += ", reqid:%s" % reqid
 
-            return None, err_msg
+            return None, err_msg, resp.status
 
-        return ret, None
+        return ret, None, resp.status
 
     def call_with_multipart(self, path, fields=None, files=None):
         """
@@ -75,7 +88,8 @@ def call_with_form(self, path, ops):
         body = '&'.join(body)
 
         content_type = "application/x-www-form-urlencoded"
-        return self.call_with(path, body, content_type, len(body))
+        ret, err, code = self.call_with(path, body, content_type, len(body))
+        return ret, err
 
     def set_header(self, field, value):
         self._header[field] = value

qiniu/rs/rs_token.py

@@ -22,6 +22,7 @@ class PutPolicy(object):
     saveKey = None
     insertOnly = None
     detectMime = None
+    mimeLimit = None
     fsizeLimit = None
     persistentNotifyUrl = None
     persistentOps = None
@@ -65,6 +66,9 @@ def token(self, mac=None):
         if self.detectMime is not None:
             token["detectMime"] = self.detectMime
 
+        if self.mimeLimit is not None:
+            token["mimeLimit"] = self.mimeLimit
+
         if self.fsizeLimit is not None:
             token["fsizeLimit"] = self.fsizeLimit
 

qiniu/rsf.py

@@ -36,7 +36,7 @@ def list_prefix(self, bucket, prefix=None, marker=None, limit=None):
         if prefix is not None:
             ops['prefix'] = prefix
         url = '%s?%s' % ('/list', urllib.urlencode(ops))
-        ret, err = self.conn.call_with(
+        ret, err, code = self.conn.call_with(
             url, body=None, content_type='application/x-www-form-urlencoded')
         if ret and not ret.get('marker'):
             err = EOF

qiniu/test/io_test.py

@@ -63,7 +63,6 @@ def test_put_quote_key():
             data = r(100)
             key = 'a\\b\\c"你好' + r(9)
             ret, err = io.put(policy.token(), key, data)
-            print err
             assert err is None
             assert ret['key'].encode('utf8') == key
 
@@ -114,7 +113,7 @@ def test_put_StringIO():
 
         def test_put_urlopen():
             key = "test_%s" % r(9)
-            data = urllib.urlopen('http://cheneya.qiniudn.com/hello_jpg')
+            data = urllib.urlopen('http://pythonsdk.qiniudn.com/hello.jpg')
             ret, err = io.put(policy.token(), key, data)
             assert err is None
             assert ret['key'] == key
@@ -178,6 +177,17 @@ def test_put_fail_reqid(self):
         ret, err = io.put("", key, data, extra)
         assert "reqid" in err
 
+    def test_put_with_uphost2(self):
+        conf.UP_HOST = "api.qiniu.com"  # mistake up host
+        localfile = "%s" % __file__
+        key = "test_up2_%s" % r(9)
+
+        extra.check_crc = 1
+        ret, err = io.put_file(policy.token(), key, localfile, extra)
+        assert err is None
+        assert ret['key'] == key
+        conf.UP_HOST = "up.qiniu.com"
+
 
 class Test_get_file_crc32(unittest.TestCase):