Merge pull request #38 from aliyun/develop

four bug fixes
aliyun · Nov 17, 2017 · 3e4cd25 · 3e4cd25
2 parents cfafaa6 + 12e95c6
commit 3e4cd25
Show file tree

Hide file tree

Showing 23 changed files with 399 additions and 47 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,13 @@
 # ChangeLog - Aliyun OSS SDK for C# 
 
+## 版本号：2.5.4  日期：2017/11/17
+### 变更内容
+- 修复：多线程调用BeginPutObject报ObjectDisposedException的问题
+- 修复：错误重试时报签名异常的问题
+- 修复：大文件上传Hang住的问题
+- 修复：非Seekable的Stream无法上传的问题
+- 修复：并发上传占用内存大的问题
+
 ## 版本号：2.5.3  日期：2017/09/22
 ### 变更内容
 - 修复：SetBucketCors当AllowedMethod为DELETE时报错的问题

diff --git a/README-CN.md b/README-CN.md
@@ -12,7 +12,7 @@
  - OSS C# SDK[在线文档](http://gosspublic.alicdn.com/AliyunNetSDK/apidocs/latest/index.html)。
 
 ## 版本
- - 当前版本：2.5.3
+ - 当前版本：2.5.4
 
 ## 运行环境
 

diff --git a/README.md b/README.md
@@ -12,7 +12,7 @@
  - OSS C# SDK[Online Documentation](http://gosspublic.alicdn.com/AliyunNetSDK/international/apidocs/latest/index.html). 
 
 ## Version
- - Current version: 2.5.3.
+ - Current version: 2.5.4.
 
 ## Run environment
 

diff --git a/samples/Properties/AssemblyInfo.cs b/samples/Properties/AssemblyInfo.cs
@@ -28,7 +28,7 @@
 //
 // You can specify all the values or you can use the default the Revision and 
 // Build Numbers by using the '*' as shown below:
-[assembly: AssemblyVersion("2.5.3")]
+[assembly: AssemblyVersion("2.5.4")]
 [assembly: NeutralResourcesLanguage("zh-CN")]
-[assembly: AssemblyFileVersion("2.5.3")]
+[assembly: AssemblyFileVersion("2.5.4")]
 
diff --git a/sdk/Commands/OssCommand.cs b/sdk/Commands/OssCommand.cs
@@ -57,11 +57,23 @@ protected virtual Stream Content
             get { return null; }
         }
 
+        protected bool UseChunkedEncoding
+        {
+            get;
+            private set;
+        }
+
         protected OssCommand(IServiceClient client, Uri endpoint, ExecutionContext context)
+            : this(client, endpoint, context, false)
+        {
+        }
+
+        protected OssCommand(IServiceClient client, Uri endpoint, ExecutionContext context, bool useChunkedEncoding)
         {
             Client = client;
             Endpoint = endpoint;
             Context = context;
+            UseChunkedEncoding = useChunkedEncoding;
         }
 
         public ServiceResponse Execute()
@@ -91,7 +103,8 @@ private ServiceRequest BuildRequest()
             {
                 Method = Method,
                 Endpoint = OssUtils.MakeBucketEndpoint(Endpoint, Bucket, conf),
-                ResourcePath = OssUtils.MakeResourcePath(Endpoint, Bucket, Key)
+                ResourcePath = OssUtils.MakeResourcePath(Endpoint, Bucket, Key),
+                UseChunkedEncoding = UseChunkedEncoding
             };
 
             foreach (var p in Parameters) 
@@ -122,7 +135,14 @@ internal abstract class OssCommand<T> : OssCommand
 
         protected OssCommand(IServiceClient client, Uri endpoint, ExecutionContext context,
                           IDeserializer<ServiceResponse, T> deserializer)
-            : base(client, endpoint, context)
+            : this(client, endpoint, context, deserializer, false)
+        {
+        }
+
+        protected OssCommand(IServiceClient client, Uri endpoint, ExecutionContext context,
+                          IDeserializer<ServiceResponse, T> deserializer, bool useChunkedEncoding)
+            : base(client, endpoint, context, useChunkedEncoding)
+
         {
             _deserializer = deserializer;
             Context.Command = this;

diff --git a/sdk/Commands/PutObjectCommand.cs b/sdk/Commands/PutObjectCommand.cs
@@ -43,7 +43,7 @@ protected override bool LeaveResponseOpen
         private PutObjectCommand(IServiceClient client, Uri endpoint, ExecutionContext context,
                                  IDeserializer<ServiceResponse, PutObjectResult> deserializer,
                                  PutObjectRequest putObjectRequest)
-            : base(client, endpoint, context, deserializer)
+            : base(client, endpoint, context, deserializer, putObjectRequest.UseChunkedEncoding)
         {
             _putObjectRequest = putObjectRequest;
         }
@@ -99,7 +99,6 @@ public static PutObjectCommand Create(IServiceClient client, Uri endpoint,
 
             var conf = OssUtils.GetClientConfiguration(client);
             var originalStream = putObjectRequest.Content;
-            var streamLength = originalStream.Length;
 
             // setup progress
             var callback = putObjectRequest.StreamTransferProgress;
@@ -112,6 +111,7 @@ public static PutObjectCommand Create(IServiceClient client, Uri endpoint,
             // wrap input stream in MD5Stream
             if (conf.EnalbeMD5Check) 
             {
+                var streamLength = originalStream.CanSeek ? originalStream.Length : -1;
                 var hashStream = new MD5Stream(originalStream, null, streamLength);
                 putObjectRequest.Content = hashStream;
                 context.ResponseHandlers.Add(new MD5DigestCheckHandler(hashStream));

diff --git a/sdk/Common/ClientConfiguration.cs b/sdk/Common/ClientConfiguration.cs
@@ -28,6 +28,7 @@ public class ClientConfiguration
         private bool _isCname = false;
         private bool _enalbeMD5Check = false;
         private long _progressUpdateInterval = 1024 * 4;
+        private long _directWriteStreamThreshold = 1024 * 1024 * 8;
 
         /// <summary>
         /// Max Http connection connection count. By default it's 512.
@@ -155,6 +156,24 @@ public void SetCustomEpochTicks(long epochTicks)
         /// </summary>
         public long TickOffset { get; internal set; }
 
+        /// <summary>
+        /// Gets or sets the direct write stream threshold.
+        /// The theshold is the file size threshold that when the uploading file size is more than this value, the HttpWebRequest will not use write buffer to save the memory.
+        /// </summary>
+        /// <value>The direct write stream threshold.</value>
+        public long DirectWriteStreamThreshold
+        {
+            get
+            {
+                return _directWriteStreamThreshold;
+            }
+
+            set
+            {
+                _directWriteStreamThreshold = value;
+            }
+        }
+
         /// <summary>
         /// Gets the default user agent
         /// </summary>

diff --git a/sdk/Common/Communication/ServiceClientImpl.cs b/sdk/Common/Communication/ServiceClientImpl.cs
@@ -36,9 +36,22 @@ public class HttpAsyncResult : AsyncResult<ServiceResponse>
 
             public ExecutionContext Context { get; set; }
 
+            internal ServiceRequest Request { get; set; }
+
             public HttpAsyncResult(AsyncCallback callback, object state)
                 : base(callback, state)
             { }
+
+            protected override void Dispose(bool disposing)
+            {
+                base.Dispose(disposing);
+
+                if (disposing && Request != null)
+                {
+                    Request.Dispose();
+                    Request = null;
+                }
+            }
         }
 
         /// <summary>
@@ -156,7 +169,7 @@ protected override ServiceResponse SendCore(ServiceRequest serviceRequest,
                                                     ExecutionContext context)
         {
             var request = HttpFactory.CreateWebRequest(serviceRequest, Configuration);
-            SetRequestContent(request, serviceRequest, false, null);
+            SetRequestContent(request, serviceRequest, false, null, Configuration);
             try
             {
                 var response = request.GetResponse() as HttpWebResponse;
@@ -177,11 +190,12 @@ protected override IAsyncResult BeginSendCore(ServiceRequest serviceRequest,
             var asyncResult = new HttpAsyncResult(callback, state)
             {
                 WebRequest = request, 
-                Context = context
+                Context = context,
+                Request = serviceRequest
             };
 
             SetRequestContent(request, serviceRequest, true,
-                              () => request.BeginGetResponse(OnGetResponseCompleted, asyncResult));
+                              () => request.BeginGetResponse(OnGetResponseCompleted, asyncResult), Configuration);
 
             return asyncResult;
         }
@@ -223,7 +237,7 @@ private void OnGetResponseCompleted(IAsyncResult ar)
         }
 
         private static void SetRequestContent(HttpWebRequest webRequest, ServiceRequest serviceRequest,
-                                              bool async, OssAction asyncCallback)
+                                              bool async, OssAction asyncCallback, ClientConfiguration clientConfiguration)
         {
             var data = serviceRequest.BuildRequestContent();
 
@@ -234,18 +248,30 @@ private static void SetRequestContent(HttpWebRequest webRequest, ServiceRequest
                 // Skip setting content body in this case.
                 if (async)
                     asyncCallback();
-                
+
                 return;
             }
 
             // Write data to the request stream.
             long userSetContentLength = -1;
             if (serviceRequest.Headers.ContainsKey(HttpHeaders.ContentLength))
                 userSetContentLength = long.Parse(serviceRequest.Headers[HttpHeaders.ContentLength]);
-
-            long streamLength = data.Length - data.Position;
-            webRequest.ContentLength = (userSetContentLength >= 0 && 
-                userSetContentLength <= streamLength) ? userSetContentLength : streamLength;
+
+            if (serviceRequest.UseChunkedEncoding || !data.CanSeek) // when data cannot seek, we have to use chunked encoding as there's no way to set the length
+            {
+                webRequest.SendChunked = true;
+                webRequest.AllowWriteStreamBuffering = false; // when using chunked encoding, the data is likely big and thus not use write buffer;
+            }
+            else
+            {
+                long streamLength = data.Length - data.Position;
+                webRequest.ContentLength = (userSetContentLength >= 0 &&
+                    userSetContentLength <= streamLength) ? userSetContentLength : streamLength;
+                if (webRequest.ContentLength > clientConfiguration.DirectWriteStreamThreshold)
+                {
+                    webRequest.AllowWriteStreamBuffering = false;
+                }
+            }
 
             if (async)
             {
@@ -254,7 +280,14 @@ private static void SetRequestContent(HttpWebRequest webRequest, ServiceRequest
                     {
                         using (var requestStream = webRequest.EndGetRequestStream(ar))
                         {
-                            IoUtils.WriteTo(data, requestStream, webRequest.ContentLength);
+                            if (!webRequest.SendChunked)
+                            {
+                                IoUtils.WriteTo(data, requestStream, webRequest.ContentLength);
+                            }
+                            else
+                            {
+                                IoUtils.WriteTo(data, requestStream);   
+                            }
                         }
                         asyncCallback();
                     }, null);
@@ -263,7 +296,14 @@ private static void SetRequestContent(HttpWebRequest webRequest, ServiceRequest
             {
                 using (var requestStream = webRequest.GetRequestStream())
                 {
-                    IoUtils.WriteTo(data, requestStream, webRequest.ContentLength);
+                    if (!webRequest.SendChunked)
+                    {
+                        IoUtils.WriteTo(data, requestStream, webRequest.ContentLength);
+                    }
+                    else
+                    {
+                        IoUtils.WriteTo(data, requestStream);
+                    }
                 }
             }
         }
@@ -311,6 +351,7 @@ private static void SetRequestHeaders(HttpWebRequest webRequest, ServiceRequest
                                               ClientConfiguration configuration)
         {
             webRequest.Timeout = configuration.ConnectionTimeout;
+            webRequest.ReadWriteTimeout = configuration.ConnectionTimeout;
             webRequest.Method = serviceRequest.Method.ToString().ToUpperInvariant();
 
             // Because it is not allowed to set common headers

diff --git a/sdk/Common/Communication/ServiceRequest.cs b/sdk/Common/Communication/ServiceRequest.cs
@@ -49,7 +49,18 @@ public bool IsRepeatable
         {
             get { return Content == null || Content.CanSeek; }
         }
-
+
+        /// <summary>
+        /// Gets or sets a value indicating whether this <see cref="T:Aliyun.OSS.Common.Communication.ServiceRequest"/>
+        /// use chunked encoding.
+        /// </summary>
+        /// <value><c>true</c> if use chunked encoding; otherwise, <c>false</c>.</value>
+        public bool UseChunkedEncoding
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Build the request URI from the request message.
         /// </summary>

diff --git a/sdk/Domain/PutObjectRequest.cs b/sdk/Domain/PutObjectRequest.cs
@@ -38,6 +38,16 @@ public Stream Content
             set { this._inputStream = value; }
         }
 
+        /// <summary>
+        /// Gets or sets a value indicating whether this <see cref="T:Aliyun.OSS.PutObjectRequest"/> use chunked encoding.
+        /// </summary>
+        /// <value><c>true</c> if use chunked encoding; otherwise, <c>false</c>.</value>
+        public bool UseChunkedEncoding
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Gets or sets the transfer progress callback
         /// </summary>
@@ -70,11 +80,23 @@ public PutObjectRequest(string bucketName, string key, Stream content)
         /// <param name="content">content to upload</param>
         /// <param name="metadata">metadata to set</param>
         public PutObjectRequest(string bucketName, string key, Stream content, ObjectMetadata metadata)
+            : this(bucketName, key, content, metadata, false) {}
+
+        /// <summary>
+        /// Puts the object result.
+        /// </summary>
+        /// <param name="bucketName">Bucket name.</param>
+        /// <param name="key">Key.</param>
+        /// <param name="content">Content.</param>
+        /// <param name="metadata">Metadata.</param>
+        /// <param name="useChunkedEncoding">If set to <c>true</c> use chunked encoding.</param>
+        public PutObjectRequest(string bucketName, string key, Stream content, ObjectMetadata metadata, bool useChunkedEncoding)
         {
             BucketName = bucketName;
             Key = key;
             Content = content;
             Metadata = metadata;
+            UseChunkedEncoding = useChunkedEncoding;
         }
 
         internal void Populate(IDictionary<string, string> headers)

diff --git a/sdk/OssClient.cs b/sdk/OssClient.cs
@@ -595,10 +595,8 @@ public IAsyncResult BeginPutObject(string bucketName, string key, string fileToU
             SetContentTypeIfNull(key, fileToUpload, ref metadata);
 
             IAsyncResult result;
-            using (Stream content = File.OpenRead(fileToUpload))
-            {
-                result = BeginPutObject(bucketName, key, content, metadata, callback, state);
-            }
+            Stream content = File.OpenRead(fileToUpload); // content will be disposed after EndPutObject is called;
+            result = BeginPutObject(bucketName, key, content, metadata, callback, state);
             return result;
         }
 

diff --git a/sdk/Properties/AssemblyInfo.cs b/sdk/Properties/AssemblyInfo.cs
@@ -31,11 +31,11 @@
 //
 // Major.Minor.Build.Revision
 //
-[assembly: AssemblyVersion("2.5.3")]
+[assembly: AssemblyVersion("2.5.4")]
 
 // The asembly is designed as CLS compliant to support CLS-compliant languages.
 //[assembly: CLSCompliant(true)]
 
 [assembly: InternalsVisibleTo("Aliyun.OSS.UnitTest, PublicKey=002400000480000094000000060200000024000052534131000400000100010045C2B8CBBFE7B414DEE24D990688805C04B57ABB8292CEC3CFBCF4C7F6BD8254C8DDEA76F8EA035D106914678AAE9DB8BA4BF1669637043DBE62E1DE2B978729CF6F3DD0080AC2209559371D26219B50309EFDA1D51800DE052B0A45C7C9238884EEA4E7DC3C595B4930785A33A90ED4A6869285C3C04AD95245C0DFC00D24CC")]
-[assembly: AssemblyFileVersion("2.5.3")]
+[assembly: AssemblyFileVersion("2.5.4")]
 
diff --git a/sdk/Util/OssRequestSigner.cs b/sdk/Util/OssRequestSigner.cs
@@ -32,8 +32,10 @@ public void Sign(ServiceRequest request, ICredentials credentials)
             {
                 var canonicalString = SignUtils.BuildCanonicalString(httpMethod, resourcePath, request);
                 var signature = ServiceSignature.Create().ComputeSignature(accessKeySecret, canonicalString);
-
-                request.Headers.Add(HttpHeaders.Authorization, "OSS " + accessKeyId + ":" + signature);
+
+                // request could be retried and the Authorization header may exist already.
+                // Fix for #OSS-1579/11349300 
+                request.Headers[HttpHeaders.Authorization] =  "OSS " + accessKeyId + ":" + signature;
             }
         }
     }

diff --git a/sdk/Util/OssUtils.cs b/sdk/Util/OssUtils.cs
@@ -345,10 +345,12 @@ internal static IAsyncResult BeginOperationHelper<TCommand>(TCommand cmd, AsyncC
         internal static TResult EndOperationHelper<TResult>(IServiceClient serviceClient, IAsyncResult asyncResult)
         {
             var response = EndOperationHelper(serviceClient, asyncResult);
-            var retryableAsyncResult = asyncResult as RetryableAsyncResult;
-            Debug.Assert(retryableAsyncResult != null);
-            OssCommand<TResult> cmd = (OssCommand<TResult>)retryableAsyncResult.Context.Command;
-            return cmd.DeserializeResponse(response);
+            using (RetryableAsyncResult retryableAsyncResult = asyncResult as RetryableAsyncResult)
+            {
+                Debug.Assert(retryableAsyncResult != null);
+                OssCommand<TResult> cmd = (OssCommand<TResult>)retryableAsyncResult.Context.Command;
+                return cmd.DeserializeResponse(response);
+            }
         }
 
         private static ServiceResponse EndOperationHelper(IServiceClient serviceClient, IAsyncResult asyncResult)