Merge pull request #384 from YangSen-qn/develop

UC Query Add SingleFlight & connect check

Author: bachue

CHANGELOG.md

@@ -1,4 +1,9 @@
 #Changelog
+## 8.1.2(2021-01-18)
+## 增加
+- 区域查询采用SingleFlight模式
+- 增加网络链接状态检测
+
 ## 8.1.1(2021-01-06)
 ## 优化
 - 优化日志统计
@@ -8,6 +13,7 @@
 - 支持分片V2
 - 支持自定义meta
 - 优化并发上传
+- 解决一些并发分片上传的野指针问题
 
 ## 8.0.5(2020-11-24)
 ## 优化

Qiniu.podspec

@@ -1,6 +1,6 @@
 Pod::Spec.new do |s|
   s.name         = 'Qiniu'
-  s.version      = '8.1.1'
+  s.version      = '8.1.2'
   s.summary      = 'Qiniu Resource Storage SDK for iOS and Mac'
   s.homepage     = 'https://github.com/qiniu/objc-sdk'
   s.social_media_url = 'http://weibo.com/qiniutek'

QiniuSDK.xcodeproj/project.pbxproj

@@ -14,6 +14,8 @@
 #import "QNUpToken.h"
 #import "QNResponseInfo.h"
 #import "QNFixedZone.h"
+#import "QNSingleFlight.h"
+
 
 @interface QNAutoZoneCache : NSObject
 @property(nonatomic, strong)NSMutableDictionary *cache;
@@ -69,6 +71,16 @@ - (QNZonesInfo *)zonesInfoForKey:(NSString *)cacheKey{
 
 @end
 
+@interface QNUCQuerySingleFlightValue : NSObject
+
+@property(nonatomic, strong)QNResponseInfo *responseInfo;
+@property(nonatomic, strong)NSDictionary *response;
+@property(nonatomic, strong)QNUploadRegionRequestMetrics *metrics;
+
+@end
+@implementation QNUCQuerySingleFlightValue
+@end
+
 @interface QNAutoZone()
 
 @property(nonatomic, strong)NSMutableDictionary *cache;
@@ -78,6 +90,15 @@ @interface QNAutoZone()
 @end
 @implementation QNAutoZone
 
++ (QNSingleFlight *)UCQuerySingleFlight {
+    static QNSingleFlight *singleFlight = nil;
+    static dispatch_once_t onceToken;
+    dispatch_once(&onceToken, ^{
+        singleFlight = [[QNSingleFlight alloc] init];
+    });
+    return singleFlight;
+}
+
 - (instancetype)init{
     if (self = [super init]) {
         _cache = [NSMutableDictionary new];
@@ -122,24 +143,43 @@ - (void)preQuery:(QNUpToken *)token
         ret(0, [QNResponseInfo successResponse], nil);
         return;
     }
-
-    QNRequestTransaction *transaction = [self createUploadRequestTransaction:token];
 
     kQNWeakSelf;
-    kQNWeakObj(transaction);
-    [transaction queryUploadHosts:^(QNResponseInfo * _Nullable responseInfo, QNUploadRegionRequestMetrics * _Nullable metrics, NSDictionary * _Nullable response) {
+    QNSingleFlight *singleFlight = [QNAutoZone UCQuerySingleFlight];
+    [singleFlight perform:token.index action:^(QNSingleFlightComplete  _Nonnull complete) {
         kQNStrongSelf;
-        kQNStrongObj(transaction);
+        QNRequestTransaction *transaction = [self createUploadRequestTransaction:token];
+        
+        kQNWeakSelf;
+        kQNWeakObj(transaction);
+        [transaction queryUploadHosts:^(QNResponseInfo * _Nullable responseInfo, QNUploadRegionRequestMetrics * _Nullable metrics, NSDictionary * _Nullable response) {
+            kQNStrongSelf;
+            kQNStrongObj(transaction);
+            
+            QNUCQuerySingleFlightValue *value = [[QNUCQuerySingleFlightValue alloc] init];
+            value.responseInfo = responseInfo;
+            value.response = response;
+            value.metrics = metrics;
+            complete(value, nil);
+
+            [self destroyUploadRequestTransaction:transaction];
+        }];
 
-        if (responseInfo.isOK) {
+    } complete:^(id  _Nullable value, NSError * _Nullable error) {
+        kQNStrongSelf;
+        
+        QNResponseInfo *responseInfo = [(QNUCQuerySingleFlightValue *)value responseInfo];
+        NSDictionary *response = [(QNUCQuerySingleFlightValue *)value response];
+        QNUploadRegionRequestMetrics *metrics = [(QNUCQuerySingleFlightValue *)value metrics];
+
+        if (responseInfo && responseInfo.isOK) {
             QNZonesInfo *zonesInfo = [QNZonesInfo infoWithDictionary:response];
             [self.lock lock];
             [self.cache setValue:zonesInfo forKey:cacheKey];
             [self.lock unlock];
             [[QNAutoZoneCache share] cache:response forKey:cacheKey];
             ret(0, responseInfo, metrics);
         } else {
-            
             if (responseInfo.isConnectionBroken) {
                 ret(kQNNetworkError, responseInfo, metrics);
             } else {
@@ -150,7 +190,6 @@ - (void)preQuery:(QNUpToken *)token
                 ret(0, responseInfo, metrics);
             }
         }
-        [self destroyUploadRequestTransaction:transaction];
     }];
 }
 

QiniuSDK/Common/QNAutoZone.m

@@ -0,0 +1,21 @@
+//
+//  QNConnectChecker.h
+//  QiniuSDK_Mac
+//
+//  Created by yangsen on 2021/1/8.
+//  Copyright © 2021 Qiniu. All rights reserved.
+//
+
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface QNConnectChecker : NSObject
+
++ (BOOL)check;
+
++ (void)check:(void(^)(BOOL isConnected))complete;
+
+@end
+
+NS_ASSUME_NONNULL_END

QiniuSDK/Http/ConnectCheck/QNConnectChecker.h

@@ -0,0 +1,114 @@
+//
+//  QNConnectChecker.m
+//  QiniuSDK_Mac
+//
+//  Created by yangsen on 2021/1/8.
+//  Copyright © 2021 Qiniu. All rights reserved.
+//
+
+#import "QNDefine.h"
+#import "QNLogUtil.h"
+#import "QNConfiguration.h"
+#import "QNSingleFlight.h"
+#import "QNConnectChecker.h"
+#import "QNUploadSystemClient.h"
+
+@interface QNConnectChecker()
+
+@end
+@implementation QNConnectChecker
+
++ (QNSingleFlight *)singleFlight {
+    static QNSingleFlight *singleFlight = nil;
+    static dispatch_once_t onceToken;
+    dispatch_once(&onceToken, ^{
+        singleFlight = [[QNSingleFlight alloc] init];
+    });
+    return singleFlight;
+}
+
++ (BOOL)check {
+    __block BOOL isConnected = false;
+    dispatch_semaphore_t semaphore = dispatch_semaphore_create(0);
+    [self check:^(BOOL isConnectedP) {
+        isConnected = isConnectedP;
+        dispatch_semaphore_signal(semaphore);
+    }];
+    dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER);
+    return isConnected;
+}
+
++ (void)check:(void (^)(BOOL isConnected))complete {
+    QNSingleFlight *singleFlight = [self singleFlight];
+    
+    kQNWeakSelf;
+    [singleFlight perform:@"connect_check" action:^(QNSingleFlightComplete  _Nonnull singleFlightComplete) {
+        kQNStrongSelf;
+        
+        [self checkAllHosts:^(BOOL isConnected) {
+            singleFlightComplete(@(isConnected), nil);
+        }];
+        
+    } complete:^(id  _Nullable value, NSError * _Nullable error) {
+        if (complete) {
+            complete([(NSNumber *)value boolValue]);
+        }
+    }];
+}
+
+
++ (void)checkAllHosts:(void (^)(BOOL isConnected))complete {
+    
+    __block int completeCount = 0;
+    __block BOOL isCompleted = false;
+    __block BOOL isConnected = false;
+    kQNWeakSelf;
+    NSArray *allHosts = [kQNGlobalConfiguration.connectCheckURLStrings copy];
+    for (NSString *host in allHosts) {
+        [self checkHost:host complete:^(BOOL isHostConnected) {
+            kQNStrongSelf;
+            
+            @synchronized (self) {
+                completeCount += 1;
+            }
+            if (isHostConnected) {
+                isConnected = YES;
+            }
+            if (isHostConnected || completeCount == allHosts.count) {
+                @synchronized (self) {
+                    if (isCompleted) {
+                        QNLogInfo(@"== check all hosts has completed totalCount:%d completeCount:%d", allHosts.count, completeCount);
+                        return;
+                    } else {
+                        QNLogInfo(@"== check all hosts completed totalCount:%d completeCount:%d", allHosts.count, completeCount);
+                        isCompleted = true;
+                    }
+                }
+                complete(isConnected);
+            } else {
+                QNLogInfo(@"== check all hosts not completed totalCount:%d completeCount:%d", allHosts.count, completeCount);
+            }
+        }];
+    }
+}
+
++ (void)checkHost:(NSString *)host complete:(void (^)(BOOL isConnected))complete {
+    
+    NSMutableURLRequest *request = [[NSMutableURLRequest alloc] init];
+    request.URL = [NSURL URLWithString:host];
+    request.HTTPMethod = @"HEAD";
+    request.timeoutInterval = kQNGlobalConfiguration.connectCheckTimeout;
+    
+    QNUploadSystemClient *client = [[QNUploadSystemClient alloc] init];
+    [client request:request connectionProxy:nil progress:nil complete:^(NSURLResponse * response, QNUploadSingleRequestMetrics * metrics, NSData * _Nullable data, NSError * error) {
+        if (response && [(NSHTTPURLResponse *)response statusCode] > 99) {
+            QNLogInfo(@"== checkHost:%@ result: true", host);
+            complete(true);
+        } else {
+            QNLogInfo(@"== checkHost:%@ result: false", host);
+            complete(false);
+        }
+    }];
+}
+
+@end

QiniuSDK/Http/ConnectCheck/QNConnectChecker.m

@@ -262,8 +262,7 @@ - (BOOL)canConnectToHost{
 
 - (BOOL)isHostUnavailable{
     // 基本不可恢复，注：会影响下次请求，范围太大可能会造成大量的timeout
-    if ((_statusCode >= -2000 && _statusCode <= -1200)
-        || _statusCode == 502 || _statusCode == 503 || _statusCode == 504 || _statusCode == 599) {
+    if (_statusCode == 502 || _statusCode == 503 || _statusCode == 504 || _statusCode == 599) {
         return true;
     } else {
         return false;

QiniuSDK/Http/QNResponseInfo.m

@@ -194,9 +194,10 @@ - (instancetype)init{
 }
 
 - (NSNumber *)totalElapsedTime{
-    if (self.metricsInfo) {
+    NSDictionary *metricsInfo = [self syncCopyMetricsInfo];
+    if (metricsInfo) {
         double time = 0;
-        for (QNUploadRegionRequestMetrics *metrics in self.metricsInfo.allValues) {
+        for (QNUploadRegionRequestMetrics *metrics in metricsInfo.allValues) {
             time += metrics.totalElapsedTime.doubleValue;
         }
         return time > 0 ? @(time) : nil;
@@ -206,9 +207,10 @@ - (NSNumber *)totalElapsedTime{
 }
 
 - (NSNumber *)requestCount{
-    if (self.metricsInfo) {
+    NSDictionary *metricsInfo = [self syncCopyMetricsInfo];
+    if (metricsInfo) {
         NSInteger count = 0;
-        for (QNUploadRegionRequestMetrics *metrics in self.metricsInfo.allValues) {
+        for (QNUploadRegionRequestMetrics *metrics in metricsInfo.allValues) {
             count += metrics.requestCount.integerValue;
         }
         return @(count);
@@ -218,9 +220,10 @@ - (NSNumber *)requestCount{
 }
 
 - (NSNumber *)bytesSend{
-    if (self.metricsInfo) {
+    NSDictionary *metricsInfo = [self syncCopyMetricsInfo];
+    if (metricsInfo) {
         long long bytes = 0;
-        for (QNUploadRegionRequestMetrics *metrics in self.metricsInfo.allValues) {
+        for (QNUploadRegionRequestMetrics *metrics in metricsInfo.allValues) {
             bytes += metrics.bytesSend.longLongValue;
         }
         return @(bytes);
@@ -230,9 +233,10 @@ - (NSNumber *)bytesSend{
 }
 
 - (NSNumber *)regionCount{
-    if (self.metricsInfo) {
+    NSDictionary *metricsInfo = [self syncCopyMetricsInfo];
+    if (metricsInfo) {
         int count = 0;
-        for (QNUploadRegionRequestMetrics *metrics in self.metricsInfo.allValues) {
+        for (QNUploadRegionRequestMetrics *metrics in metricsInfo.allValues) {
             if (![metrics.region.zoneInfo.regionId isEqualToString:QNZoneInfoEmptyRegionId]) {
                 count += 1;
             }
@@ -258,5 +262,11 @@ - (void)addMetrics:(QNUploadRegionRequestMetrics *)metrics{
     }
 }
 
+- (NSDictionary<NSString *, QNUploadRegionRequestMetrics *> *)syncCopyMetricsInfo {
+    @synchronized (self) {
+        return [_metricsInfo copy];
+    }
+}
+
 
 @end

QiniuSDK/Http/QNUploadRequestMetrics.m

@@ -18,6 +18,7 @@
 #import "QNResponseInfo.h"
 #import "QNRequestClient.h"
 
+#import "QNConnectChecker.h"
 #import "QNDnsPrefetch.h"
 
 #import "QNReportItem.h"
@@ -145,6 +146,11 @@ - (void)retryRequest:(NSURLRequest *)request
                                                                response:(NSHTTPURLResponse *)response
                                                                    body:responseData
                                                                   error:error];
+        if ([self shouldCheckConnect:responseInfo] && ![QNConnectChecker check]) {
+            NSString *message = [NSString stringWithFormat:@"check origin statusCode:%d error:%@", responseInfo.statusCode, responseInfo.error];
+            responseInfo = [QNResponseInfo errorResponseInfo:NSURLErrorNotConnectedToInternet errorDesc:message];
+        }
+        
         QNLogInfo(@"key:%@ response:%@", self.requestInfo.key, responseInfo);
         if (shouldRetry(responseInfo, responseDic)
             && self.currentRetryTime < self.config.retryMax
@@ -160,6 +166,16 @@ - (void)retryRequest:(NSURLRequest *)request
 
 }
 
+- (BOOL)shouldCheckConnect:(QNResponseInfo *)responseInfo {
+    return responseInfo.statusCode == kQNNetworkError ||
+    responseInfo.statusCode == -1001 /* NSURLErrorTimedOut */ ||
+    responseInfo.statusCode == -1003 /* NSURLErrorCannotFindHost */ ||
+    responseInfo.statusCode == -1004 /* NSURLErrorCannotConnectToHost */ ||
+    responseInfo.statusCode == -1005 /* NSURLErrorNetworkConnectionLost */ ||
+    responseInfo.statusCode == -1006 /* NSURLErrorDNSLookupFailed */ ||
+    responseInfo.statusCode == -1009 /* NSURLErrorNotConnectedToInternet */;
+}
+
 - (void)complete:(QNResponseInfo *)responseInfo
           server:(id <QNUploadServer>)server
         response:(NSDictionary *)response

QiniuSDK/Http/Request/QNHttpSingleRequest.m

@@ -16,7 +16,7 @@ typedef void (^QNRequestClientCompleteHandler)(NSURLResponse * _Nullable, QNUplo
 
 - (void)request:(NSURLRequest *)request
 connectionProxy:(NSDictionary * _Nullable)connectionProxy
-       progress:(void(^)(long long totalBytesWritten, long long totalBytesExpectedToWrite))progress
+       progress:(void(^ _Nullable)(long long totalBytesWritten, long long totalBytesExpectedToWrite))progress
        complete:(QNRequestClientCompleteHandler)complete;
 
 - (void)cancel;