Merge pull request #174 from aliyun/release_1.91.6

release 1.91.6

Author: FangQianan

.gitignore

@@ -81,6 +81,7 @@ src/ossfs
 src/test_curl_util
 src/test_page_list
 src/test_string_util
+src/test_mempool
 test/chaos-http-proxy-*
 test/junk_data
 test/s3proxy-*

ChangeLog

@@ -1,5 +1,17 @@
 ChangeLog for OSSFS
 ------------------
+## v1.91.6 (17/03/2025)
+ - Print information to suggest users to mount with OSS V4 signature.
+ - Optimized the prefetch-task generation logic under direct-read mode to better performance.
+ - Improved concurrent IO performance.
+ - Modify the folder size in the mount point to 4096.
+ - Fixed a bug that in use_cache mode, when the ram user does not have bucket write permission, the cache file is overwritten incorrectly, resulting in data read errors.
+ - Optimized the printing of error information when specifying an incorrect passwd-file.
+ - Optimized the printing of log information when instance_name is specified.
+ - Enable noobj_cache by default.
+ - Force objects ending with '/' to be recognized as folders.
+ - Optimized memory usage in direct reading mode.
+
 ## v1.91.5 (25/12/2024)
  - Fixed some bugs that users get unexpected results on mounting with specific policies for the bucket/prefix set.
  - Fixed a bug that ossfs take a file as existing if HeadObj returns 403.

configure.ac

@@ -20,7 +20,7 @@
 dnl Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.59)
-AC_INIT(ossfs, 1.91.5)
+AC_INIT(ossfs, 1.91.6)
 AC_CONFIG_HEADER([config.h])
 
 AC_CANONICAL_SYSTEM

doc/man/ossfs.1.in

@@ -139,11 +139,10 @@ specify expire time (seconds) for entries in the stat cache and symbolic link ca
 specify expire time (seconds) for entries in the stat cache and symbolic link cache. This expire time is based on the time from the last access time of those cache.
 This option is exclusive with stat_cache_expire, and is left for compatibility with older versions.
 .TP
-\fB\-o\fR enable_noobj_cache (default is disable)
-enable cache entries for the object which does not exist.
-ossfs always has to check whether file (or sub directory) exists under object (path) when ossfs does some command, since ossfs has recognized a directory which does not exist and has files or sub directories under itself.
-It increases ListBucket request and makes performance bad.
-You can specify this option for performance, ossfs memorizes in stat cache that the object (file or directory) does not exist.
+\fB\-o\fR disable_noobj_cache (default is enable)
+By default ossfs memorizes when an object does not exist up until the stat cache timeout.
+This caching can cause staleness for applications.
+If disabled, ossfs will not memorize objects and may cause extra HeadObject requests and reduce performance.
 .TP
 \fB\-o\fR no_check_certificate (by default this option is disabled)
 server certificate won't be checked against the available certificate authorities.

scripts/ossfs-coverage-centos7.sh

@@ -37,6 +37,7 @@ DBGLEVEL=debug ALL_TESTS=1 OSSFS_CREDENTIALS_FILE=/root/.passwd-ossfs TEST_BUCKE
 ${OSSFS_SOURCE_DIR}/src/test_page_list 
 ${OSSFS_SOURCE_DIR}/src/test_curl_util
 ${OSSFS_SOURCE_DIR}/src/test_string_util
+${OSSFS_SOURCE_DIR}/src/test_mempool
 
 gcovr -r ${OSSFS_SOURCE_DIR}/src --html-details -o ${OSSFS_SOURCE_DIR}/coverage_html/coverage.html
 

src/Makefile.am

@@ -58,7 +58,8 @@ ossfs_SOURCES = \
     autolock.cpp \
     common_auth.cpp \
     threadpoolman.cpp \
-    direct_reader.cpp
+    direct_reader.cpp \
+    memorypool.cpp
 if USE_SSL_OPENSSL
     ossfs_SOURCES += openssl_auth.cpp
 endif
@@ -74,7 +75,8 @@ ossfs_LDADD = $(DEPS_LIBS)
 noinst_PROGRAMS = \
     test_curl_util \
     test_page_list \
-    test_string_util
+    test_string_util \ 
+    test_mempool
 
 test_curl_util_SOURCES = common_auth.cpp curl_util.cpp string_util.cpp test_curl_util.cpp s3fs_global.cpp s3fs_logger.cpp
 if USE_SSL_OPENSSL
@@ -98,10 +100,15 @@ test_page_list_SOURCES = \
 
 test_string_util_SOURCES = string_util.cpp test_string_util.cpp s3fs_logger.cpp
 
+test_mempool_SOURCES = memorypool.cpp test_mempool.cpp s3fs_logger.cpp s3fs_global.cpp string_util.cpp
+test_mempool_LDFLAGS = -pthread
+test_mempool_LDADD = $(DEPS_LIBS)
+
 TESTS = \
     test_curl_util \
     test_page_list \
-    test_string_util
+    test_string_util \
+    test_mempool
 
 clang-tidy:
 	clang-tidy $(ossfs_SOURCES) -- $(DEPS_CFLAGS) $(CPPFLAGS)

src/cache.cpp

@@ -149,7 +149,11 @@ bool convert_header_to_stat(const std::string& strpath, const headers_t& meta, s
     }
 
     // size
-    pst->st_size = get_size(meta);
+    if (S_ISDIR(pst->st_mode)) {
+        pst->st_size = 4096;
+    } else {
+        pst->st_size = get_size(meta);
+    }
 
     //change symlink to S_IFREG
     if (noextendedmeta && S_ISLNK(pst->st_mode) && !is_check_meta(pst->st_size, check_size_meta)) {
@@ -217,7 +221,7 @@ pthread_mutex_t StatCache::stat_cache_lock;
 //-------------------------------------------------------------------
 // Constructor/Destructor
 //-------------------------------------------------------------------
-StatCache::StatCache() : IsExpireTime(true), IsExpireIntervalType(false), ExpireTime(15 * 60), CacheSize(100000), IsCacheNoObject(false),
+StatCache::StatCache() : IsExpireTime(true), IsExpireIntervalType(false), ExpireTime(15 * 60), CacheSize(100000), IsCacheNoObject(true),
  IsNoExtendedMeta(false), CheckSizeForMeta(0LL)
 {
     if(this == StatCache::getStatCacheData()){

src/curl.cpp

@@ -868,41 +868,6 @@ sse_type_t S3fsCurl::SetSseType(sse_type_t type)
     return old;
 }
 
-bool S3fsCurl::SetSseCKeys(const char* filepath)
-{
-    if(!filepath){
-        S3FS_PRN_ERR("SSE-C keys filepath is empty.");
-        return false;
-    }
-    struct stat st;
-    if(0 != stat(filepath, &st)){
-        S3FS_PRN_ERR("could not open use_sse keys file(%s).", filepath);
-        return false;
-    }
-    if(st.st_mode & (S_IXUSR | S_IRWXG | S_IRWXO)){
-        S3FS_PRN_ERR("use_sse keys file %s should be 0600 permissions.", filepath);
-        return false;
-    }
-
-    S3fsCurl::sseckeys.clear();
-
-    std::ifstream ssefs(filepath);
-    if(!ssefs.good()){
-        S3FS_PRN_ERR("Could not open SSE-C keys file(%s).", filepath);
-        return false;
-    }
-
-    std::string line;
-    while(getline(ssefs, line)){
-        S3fsCurl::PushbackSseKeys(line);
-    }
-    if(S3fsCurl::sseckeys.empty()){
-        S3FS_PRN_ERR("There is no SSE Key in file(%s).", filepath);
-        return false;
-    }
-    return true;
-}
-
 bool S3fsCurl::SetSseKmsid(const char* kmsid)
 {
     if(!kmsid || '\0' == kmsid[0]){
@@ -2630,201 +2595,6 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
     return result;
 }
 
-//
-// Returns the Amazon AWS signature for the given parameters.
-//
-// @param method e.g., "GET"
-// @param content_type e.g., "application/x-directory"
-// @param date e.g., get_date_rfc850()
-// @param resource e.g., "/pub"
-//
-std::string S3fsCurl::CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource, const std::string& secret_access_key, const std::string& access_token)
-{
-    std::string Signature;
-    std::string StringToSign;
-
-    if(!access_token.empty()){
-        requestHeaders = curl_slist_sort_insert(requestHeaders, "x-oss-security-token", access_token.c_str());
-    }
-
-    StringToSign += method + "\n";
-    StringToSign += strMD5 + "\n";        // md5
-    StringToSign += content_type + "\n";
-    StringToSign += date + "\n";
-    StringToSign += get_canonical_headers(requestHeaders, true);
-    StringToSign += resource;
-
-    const void* key            = secret_access_key.data();
-    size_t key_len             = secret_access_key.size();
-    const unsigned char* sdata = reinterpret_cast<const unsigned char*>(StringToSign.data());
-    size_t sdata_len           = StringToSign.size();
-    unsigned char* md          = NULL;
-    unsigned int md_len        = 0;;
-
-    s3fs_HMAC(key, key_len, sdata, sdata_len, &md, &md_len);
-
-    char* base64;
-    if(NULL == (base64 = s3fs_base64(md, md_len))){
-        delete[] md;
-        return std::string("");  // ENOMEM
-    }
-    delete[] md;
-
-    Signature = base64;
-    delete[] base64;
-
-    return Signature;
-}
-
-std::string S3fsCurl::CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601, const std::string& secret_access_key, const std::string& access_token)
-{
-    std::string Signature, StringCQ, StringToSign;
-    std::string uriencode;
-
-    if(!access_token.empty()){
-        requestHeaders = curl_slist_sort_insert(requestHeaders, "x-oss-security-token", access_token.c_str());
-    }
-
-    uriencode = urlEncode(canonical_uri);
-    StringCQ  = method + "\n";
-    if(method == "HEAD" || method == "PUT" || method == "DELETE"){
-        StringCQ += uriencode + "\n";
-    }else if(method == "GET" && uriencode.empty()){
-        StringCQ +="/\n";
-    }else if(method == "GET" && is_prefix(uriencode.c_str(), "/")){
-        StringCQ += uriencode +"\n";
-    }else if(method == "GET" && !is_prefix(uriencode.c_str(), "/")){
-        StringCQ += "/\n" + urlEncode2(canonical_uri) +"\n";
-    }else if(method == "POST"){
-        StringCQ += uriencode + "\n";
-    }
-    StringCQ += urlEncode2(query_string) + "\n";
-    StringCQ += get_canonical_headers(requestHeaders) + "\n";
-    StringCQ += get_sorted_header_keys(requestHeaders) + "\n";
-    StringCQ += payload_hash;
-
-    std::string   kSecret = "AWS4" + secret_access_key;
-    unsigned char *kDate, *kRegion, *kService, *kSigning, *sRequest               = NULL;
-    unsigned int  kDate_len,kRegion_len, kService_len, kSigning_len, sRequest_len = 0;
-
-    s3fs_HMAC256(kSecret.c_str(), kSecret.size(), reinterpret_cast<const unsigned char*>(strdate.data()), strdate.size(), &kDate, &kDate_len);
-    s3fs_HMAC256(kDate, kDate_len, reinterpret_cast<const unsigned char*>(endpoint.c_str()), endpoint.size(), &kRegion, &kRegion_len);
-    s3fs_HMAC256(kRegion, kRegion_len, reinterpret_cast<const unsigned char*>("s3"), sizeof("s3") - 1, &kService, &kService_len);
-    s3fs_HMAC256(kService, kService_len, reinterpret_cast<const unsigned char*>("aws4_request"), sizeof("aws4_request") - 1, &kSigning, &kSigning_len);
-    delete[] kDate;
-    delete[] kRegion;
-    delete[] kService;
-
-    const unsigned char* cRequest     = reinterpret_cast<const unsigned char*>(StringCQ.c_str());
-    size_t               cRequest_len = StringCQ.size();
-    s3fs_sha256(cRequest, cRequest_len, &sRequest, &sRequest_len);
-
-    StringToSign  = "AWS4-HMAC-SHA256\n";
-    StringToSign += date8601 + "\n";
-    StringToSign += strdate + "/" + endpoint + "/s3/aws4_request\n";
-    StringToSign += s3fs_hex_lower(sRequest, sRequest_len);
-    delete[] sRequest;
-
-    const unsigned char* cscope     = reinterpret_cast<const unsigned char*>(StringToSign.c_str());
-    size_t               cscope_len = StringToSign.size();
-    unsigned char*       md         = NULL;
-    unsigned int         md_len     = 0;
-
-    s3fs_HMAC256(kSigning, kSigning_len, cscope, cscope_len, &md, &md_len);
-    delete[] kSigning;
-
-    Signature = s3fs_hex_lower(md, md_len);
-    delete[] md;
-
-    return Signature;
-}
-
-void S3fsCurl::insertV4Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token)
-{
-    std::string server_path = type == REQTYPE_LISTBUCKET ? "/" : path;
-    std::string payload_hash;
-    switch (type) {
-        case REQTYPE_PUT:
-            if(GetUnsignedPayload()){
-                payload_hash = "UNSIGNED-PAYLOAD";
-            }else{
-                payload_hash = s3fs_sha256_hex_fd(b_infile == NULL ? -1 : fileno(b_infile), 0, -1);
-            }
-            break;
-
-        case REQTYPE_COMPLETEMULTIPOST:
-            {
-                size_t          cRequest_len = strlen(reinterpret_cast<const char *>(b_postdata));
-                unsigned char*  sRequest     = NULL;
-                unsigned int    sRequest_len = 0;
-                s3fs_sha256(b_postdata, cRequest_len, &sRequest, &sRequest_len);
-                payload_hash = s3fs_hex_lower(sRequest, sRequest_len);
-                delete[] sRequest;
-                break;
-            }
-
-        case REQTYPE_UPLOADMULTIPOST:
-            if(GetUnsignedPayload()){
-                payload_hash = "UNSIGNED-PAYLOAD";
-            }else{
-                payload_hash = s3fs_sha256_hex_fd(partdata.fd, partdata.startpos, partdata.size);
-            }
-            break;
-        default:
-            break;
-    }
-
-    if(b_infile != NULL && payload_hash.empty()){
-        S3FS_PRN_ERR("Failed to make SHA256.");
-        // TODO: propagate error
-    }
-
-    S3FS_PRN_INFO3("computing signature [%s] [%s] [%s] [%s]", op.c_str(), server_path.c_str(), query_string.c_str(), payload_hash.c_str());
-    std::string strdate;
-    std::string date8601;
-    get_date_sigv3(strdate, date8601);
-
-    std::string contentSHA256 = payload_hash.empty() ? EMPTY_PAYLOAD_HASH : payload_hash;
-    const std::string realpath = pathrequeststyle ? "/" + S3fsCred::GetBucket() + server_path : server_path;
-
-    //string canonical_headers, signed_headers;
-    requestHeaders = curl_slist_sort_insert(requestHeaders, "host", get_bucket_host().c_str());
-    requestHeaders = curl_slist_sort_insert(requestHeaders, "x-oss-content-sha256", contentSHA256.c_str());
-    requestHeaders = curl_slist_sort_insert(requestHeaders, "x-oss-date", date8601.c_str());
-
-    if (S3fsCurl::IsRequesterPays()) {
-        requestHeaders = curl_slist_sort_insert(requestHeaders, "x-oss-request-payer", "requester");
-    }
-
-    if(!S3fsCurl::IsPublicBucket()){
-        std::string Signature = CalcSignature(op, realpath, query_string + (type == REQTYPE_PREMULTIPOST || type == REQTYPE_MULTILIST ? "=" : ""), strdate, contentSHA256, date8601, secret_access_key, access_token);
-        std::string auth = "AWS4-HMAC-SHA256 Credential=" + access_key_id + "/" + strdate + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature;
-        requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", auth.c_str());
-    }
-}
-
-void S3fsCurl::insertV2Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token)
-{
-    std::string resource;
-    std::string turl;
-    std::string server_path = type == REQTYPE_LISTBUCKET ? "/" : path;
-    MakeUrlResource(server_path.c_str(), resource, turl);
-    if(!query_string.empty() && type != REQTYPE_CHKBUCKET && type != REQTYPE_LISTBUCKET){
-        resource += "?" + query_string;
-    }
-
-    std::string date = get_date_rfc850();
-    requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str());
-    if(op != "PUT" && op != "POST"){
-        requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", NULL);
-    }
-
-    if(!S3fsCurl::IsPublicBucket()){
-        std::string Signature = CalcSignatureV2(op, get_header_value(requestHeaders, "Content-MD5"), get_header_value(requestHeaders, "Content-Type"), date, resource, secret_access_key, access_token);
-        requestHeaders   = curl_slist_sort_insert(requestHeaders, "Authorization", std::string("AWS " + access_key_id + ":" + Signature).c_str());
-    }
-}
-
 void S3fsCurl::insertIBMIAMHeaders(const std::string& access_key_id, const std::string& access_token)
 {
     requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", ("Bearer " + access_token).c_str());

src/curl.h

@@ -256,12 +256,8 @@ class S3fsCurl
         bool ResetHandle(bool lock_already_held = false);
         bool RemakeHandle();
         bool ClearInternalData();
-        void insertV4Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token);
-        void insertV2Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token);
         void insertIBMIAMHeaders(const std::string& access_key_id, const std::string& access_token);
         void insertAuthHeaders();
-        std::string CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource, const std::string& secret_access_key, const std::string& access_token);
-        std::string CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601, const std::string& secret_access_key, const std::string& access_token);
         int UploadMultipartPostSetup(const char* tpath, int part_num, const std::string& upload_id);
         int CopyMultipartPostSetup(const char* from, const char* to, int part_num, const std::string& upload_id, headers_t& meta);
         bool UploadMultipartPostComplete();
@@ -307,7 +303,6 @@ class S3fsCurl
         static bool IsSseCType() { return (sse_type_t::SSE_C == S3fsCurl::ssetype); }
         static bool IsSseKmsType() { return (sse_type_t::SSE_KMS == S3fsCurl::ssetype); }
         static bool FinalCheckSse();
-        static bool SetSseCKeys(const char* filepath);
         static bool SetSseKmsid(const char* kmsid);
         static bool IsSetSseKmsId() { return !S3fsCurl::ssekmsid.empty(); }
         static const char* GetSseKmsId() { return S3fsCurl::ssekmsid.c_str(); }