OAK-11267 - Upgrade Azure SDK V8 to V12 for oak-blob-azure

oak-blob-cloud-azure/pom.xml

@@ -44,11 +44,14 @@
                         <Import-Package>
                             com.fasterxml.jackson.annotation;resolution:=optional,
                             com.fasterxml.jackson.databind*;resolution:=optional,
-                            com.fasterxml.jackson.dataformat.xml;resolution:=optional,
+                            com.fasterxml.jackson.dataformat.*;resolution:=optional,
                             com.fasterxml.jackson.datatype*;resolution:=optional,
                             com.azure.identity.broker.implementation;resolution:=optional,
                             com.azure.xml;resolution:=optional,
-                            com.microsoft.aad.msal4jextensions*;resolution:=optional,
+                            com.azure.storage.common*;resolution:=optional,
+                            com.azure.storage.internal*;resolution:=optional,
+                            com.microsoft.aad.*;resolution:=optional,
+                            com.microsoft.aad.msal4jextensions.persistence*;resolution:=optional,
                             com.sun.net.httpserver;resolution:=optional,
                             sun.misc;resolution:=optional,
                             net.jcip.annotations;resolution:=optional,
@@ -72,6 +75,14 @@
                             azure-core,
                             azure-identity,
                             azure-json,
+                            azure-xml,
+                            azure-storage-blob,
+                            azure-storage-common,
+                            azure-storage-internal-avro,
+                            com.microsoft.aad,
+                            com.microsoft.aad.msal4jextensions,
+                            com.microsoft.aad.msal4jextensions.persistence,
+                            jackson-dataformat-xml,
                             guava,
                             jsr305,
                             reactive-streams,
@@ -174,6 +185,11 @@
             <groupId>com.microsoft.azure</groupId>
             <artifactId>azure-storage</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-storage-blob</artifactId>
+            <version>12.27.1</version>
+        </dependency>
         <dependency>
             <groupId>com.microsoft.azure</groupId>
             <artifactId>azure-keyvault-core</artifactId>

oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureBlobStoreBackend.java

@@ -0,0 +1,27 @@
+package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
+import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
+import org.jetbrains.annotations.NotNull;
+
+import java.net.URI;
+import java.util.Properties;
+
+
+public abstract class AbstractAzureBlobStoreBackend extends AbstractSharedBackend {
+
+    protected abstract DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options);
+    protected abstract DataRecord completeHttpUpload(@NotNull String uploadTokenStr) throws DataRecordUploadException, DataStoreException;
+    protected abstract void setHttpDownloadURIExpirySeconds(int seconds);
+    protected abstract  void setHttpUploadURIExpirySeconds(int seconds);
+    protected abstract void setHttpDownloadURICacheSize(int maxSize);
+    protected abstract URI createHttpDownloadURI(@NotNull DataIdentifier identifier, @NotNull DataRecordDownloadOptions downloadOptions);
+    public abstract void setProperties(final Properties properties);
+
+}

oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java

@@ -25,6 +25,7 @@
 import org.apache.jackrabbit.core.data.DataIdentifier;
 import org.apache.jackrabbit.core.data.DataRecord;
 import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobStoreBackendV8;
 import org.apache.jackrabbit.oak.plugins.blob.AbstractSharedCachingDataStore;
 import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider;
 import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
@@ -41,11 +42,18 @@ public class AzureDataStore extends AbstractSharedCachingDataStore implements Co
 
     protected Properties properties;
 
-    private AzureBlobStoreBackend azureBlobStoreBackend;
+    private AbstractAzureBlobStoreBackend azureBlobStoreBackend;
+
+    private final boolean useAzureSdkV12 = Boolean.getBoolean("blob.azure.v12.enabled");
 
     @Override
     protected AbstractSharedBackend createBackend() {
-        azureBlobStoreBackend = new AzureBlobStoreBackend();
+        if(useAzureSdkV12) {
+            azureBlobStoreBackend = new AzureBlobStoreBackend();
+        } else {
+            azureBlobStoreBackend = new AzureBlobStoreBackendV8();
+        }
+
         if (null != properties) {
             azureBlobStoreBackend.setProperties(properties);
         }

oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreService.java

@@ -20,6 +20,7 @@
 package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
 
 import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.jetbrains.annotations.NotNull;
 import org.osgi.service.component.annotations.Component;
 import org.osgi.service.component.annotations.ConfigurationPolicy;
 import org.osgi.service.component.annotations.Reference;
@@ -32,7 +33,7 @@ public class AzureDataStoreService extends AbstractAzureDataStoreService {
 
     public static final String NAME = "org.apache.jackrabbit.oak.plugins.blob.datastore.AzureDataStore";
 
-    protected StatisticsProvider getStatisticsProvider(){
+    protected @NotNull StatisticsProvider getStatisticsProvider(){
         return statisticsProvider;
     }
 

oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Constants.java

@@ -0,0 +1,19 @@
+package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
+
+public interface Constants {
+    String META_DIR_NAME = "META";
+    String META_KEY_PREFIX = META_DIR_NAME + "/";
+
+    String REF_KEY = "reference.key";
+    String LAST_MODIFIED_KEY = "lastModified";
+
+    long BUFFERED_STREAM_THRESHOLD = 1024 * 1024;
+    long MIN_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 10; // 10MB
+    long MAX_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 100; // 100MB
+    long MAX_SINGLE_PUT_UPLOAD_SIZE = 1024 * 1024 * 256; // 256MB, Azure limit
+    long MAX_BINARY_UPLOAD_SIZE = (long) Math.floor(1024L * 1024L * 1024L * 1024L * 4.75); // 4.75TB, Azure limit
+    int MAX_ALLOWABLE_UPLOAD_URIS = 50000; // Azure limit
+    int MAX_UNIQUE_RECORD_TRIES = 10;
+    int DEFAULT_CONCURRENT_REQUEST_COUNT = 2;
+    int MAX_CONCURRENT_REQUEST_COUNT = 50;
+}

oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java

@@ -1,130 +1,92 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
 package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetSocketAddress;
-import java.net.Proxy;
-import java.net.SocketAddress;
-import java.net.URISyntaxException;
-import java.security.InvalidKeyException;
-import java.util.Properties;
-
-import org.apache.jackrabbit.guava.common.base.Strings;
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.OperationContext;
-import com.microsoft.azure.storage.RetryExponentialRetry;
-import com.microsoft.azure.storage.RetryNoRetry;
-import com.microsoft.azure.storage.RetryPolicy;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.blob.BlobRequestOptions;
-import com.microsoft.azure.storage.blob.CloudBlobClient;
-import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.azure.core.http.HttpClient;
+import com.azure.core.http.ProxyOptions;
+import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.common.policy.RequestRetryOptions;
+import com.azure.storage.common.policy.RetryPolicyType;
+import com.google.common.base.Strings;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.jackrabbit.core.data.DataStoreException;
 import org.apache.jackrabbit.oak.commons.PropertiesUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-public final class Utils {
-
-    public static final String DEFAULT_CONFIG_FILE = "azure.properties";
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.time.Duration;
+import java.util.Properties;
 
+public class Utils {
     public static final String DASH = "-";
+    public static final String DEFAULT_CONFIG_FILE = "azure.properties";
 
-    /**
-     * private constructor so that class cannot initialized from outside.
-     */
-    private Utils() {
-    }
-
-    /**
-     * Create CloudBlobClient from properties.
-     *
-     * @param connectionString connectionString to configure @link {@link CloudBlobClient}
-     * @return {@link CloudBlobClient}
-     */
-    public static CloudBlobClient getBlobClient(@NotNull final String connectionString) throws URISyntaxException, InvalidKeyException {
-        return getBlobClient(connectionString, null);
-    }
+    public Utils() {}
 
-    public static CloudBlobClient getBlobClient(@NotNull final String connectionString,
-                                                @Nullable final BlobRequestOptions requestOptions) throws URISyntaxException, InvalidKeyException {
-        CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
-        CloudBlobClient client = account.createCloudBlobClient();
-        if (null != requestOptions) {
-            client.setDefaultRequestOptions(requestOptions);
-        }
-        return client;
-    }
+    public static BlobContainerClient getBlobContainer(String accountName, String accountKey, @NotNull final String connectionString,
+                                                       @NotNull final String containerName,
+                                                       @Nullable final RequestRetryOptions retryOptions,
+                                                       boolean isProxyNeeded, final Properties properties) throws DataStoreException {
+        BlobContainerClient blobContainerClient;
+        try {
+            if(isProxyNeeded) {
+                String proxyHost = properties.getProperty(AzureConstants.PROXY_HOST);
+                String proxyPort = properties.getProperty(AzureConstants.PROXY_PORT);
+
+                ProxyOptions proxyOptions = new ProxyOptions(ProxyOptions.Type.HTTP,
+                        new InetSocketAddress(proxyHost, Integer.parseInt(proxyPort)));
+
+                HttpClient httpClient = new NettyAsyncHttpClientBuilder()
+                        .proxy(proxyOptions)
+                        .build();
+
+                BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
+                        .connectionString(connectionString)
+                        .httpClient(httpClient)
+                        .retryOptions(retryOptions)
+                        .buildClient();
+                blobContainerClient = blobServiceClient.getBlobContainerClient(containerName);
+                return blobContainerClient;
+            }
 
-    public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
-                                                      @NotNull final String containerName) throws DataStoreException {
-        return getBlobContainer(connectionString, containerName, null);
-    }
+            BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
+                    .connectionString(connectionString)
+                    .retryOptions(retryOptions)
+                    .buildClient();
+            blobContainerClient = blobServiceClient.getBlobContainerClient(containerName);
+            return blobContainerClient;
 
-    public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
-                                                      @NotNull final String containerName,
-                                                      @Nullable final BlobRequestOptions requestOptions) throws DataStoreException {
-        try {
-            CloudBlobClient client = (
-                    (null == requestOptions)
-                            ? Utils.getBlobClient(connectionString)
-                            : Utils.getBlobClient(connectionString, requestOptions)
-            );
-            return client.getContainerReference(containerName);
-        } catch (InvalidKeyException | URISyntaxException | StorageException e) {
+        } catch (Exception e) {
             throw new DataStoreException(e);
         }
     }
 
-    public static void setProxyIfNeeded(final Properties properties) {
+    public static boolean isProxyNeeded(final Properties properties) {
         String proxyHost = properties.getProperty(AzureConstants.PROXY_HOST);
         String proxyPort = properties.getProperty(AzureConstants.PROXY_PORT);
 
-        if (!Strings.isNullOrEmpty(proxyHost) &&
-            Strings.isNullOrEmpty(proxyPort)) {
-            int port = Integer.parseInt(proxyPort);
-            SocketAddress proxyAddr = new InetSocketAddress(proxyHost, port);
-            Proxy proxy = new Proxy(Proxy.Type.HTTP, proxyAddr);
-            OperationContext.setDefaultProxy(proxy);
-        }
+        return !Strings.isNullOrEmpty(proxyHost) && Strings.isNullOrEmpty(proxyPort);
     }
 
-    public static RetryPolicy getRetryPolicy(final String maxRequestRetry) {
-        int retries = PropertiesUtil.toInteger(maxRequestRetry, -1);
-        if (retries < 0) {
+    public static RequestRetryOptions getRetryOptions(final String maxRequestRetryCount) {
+        int retries = PropertiesUtil.toInteger(maxRequestRetryCount, -1);
+        if(retries < 0) {
             return null;
         }
+
         if (retries == 0) {
-            return new RetryNoRetry();
+            return new RequestRetryOptions(RetryPolicyType.FIXED, 1, 1, null, null, null);
         }
-        return new RetryExponentialRetry(RetryPolicy.DEFAULT_CLIENT_BACKOFF, retries);
+        return new RequestRetryOptions(RetryPolicyType.EXPONENTIAL, retries, (int) Duration.ofSeconds(3).toMillis(), null, null, null);
     }
 
-
     public static String getConnectionStringFromProperties(Properties properties) {
-
         String sasUri = properties.getProperty(AzureConstants.AZURE_SAS, "");
         String blobEndpoint = properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "");
         String connectionString = properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, "");
@@ -141,7 +103,7 @@ public static String getConnectionStringFromProperties(Properties properties) {
 
         return getConnectionString(
                 accountName,
-                accountKey, 
+                accountKey,
                 blobEndpoint);
     }
 
@@ -153,15 +115,11 @@ public static String getConnectionStringForSas(String sasUri, String blobEndpoin
         }
     }
 
-    public static String getConnectionString(final String accountName, final String accountKey) {
-        return getConnectionString(accountName, accountKey, null);
-    }
-
     public static String getConnectionString(final String accountName, final String accountKey, String blobEndpoint) {
         StringBuilder connString = new StringBuilder("DefaultEndpointsProtocol=https");
         connString.append(";AccountName=").append(accountName);
         connString.append(";AccountKey=").append(accountKey);
-        
+
         if (!Strings.isNullOrEmpty(blobEndpoint)) {
             connString.append(";BlobEndpoint=").append(blobEndpoint);
         }