Skip to content

Commit 2949118

Browse files
OAK-11267 - Upgrade Azure SDK V8 to V12 for oak-blob-azure
1 parent 1294f24 commit 2949118

File tree

25 files changed

+2571
-802
lines changed

25 files changed

+2571
-802
lines changed

oak-blob-cloud-azure/pom.xml

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,14 @@
4444
<Import-Package>
4545
com.fasterxml.jackson.annotation;resolution:=optional,
4646
com.fasterxml.jackson.databind*;resolution:=optional,
47-
com.fasterxml.jackson.dataformat.xml;resolution:=optional,
47+
com.fasterxml.jackson.dataformat.*;resolution:=optional,
4848
com.fasterxml.jackson.datatype*;resolution:=optional,
4949
com.azure.identity.broker.implementation;resolution:=optional,
5050
com.azure.xml;resolution:=optional,
51-
com.microsoft.aad.msal4jextensions*;resolution:=optional,
51+
com.azure.storage.common*;resolution:=optional,
52+
com.azure.storage.internal*;resolution:=optional,
53+
com.microsoft.aad.*;resolution:=optional,
54+
com.microsoft.aad.msal4jextensions.persistence*;resolution:=optional,
5255
com.sun.net.httpserver;resolution:=optional,
5356
sun.misc;resolution:=optional,
5457
net.jcip.annotations;resolution:=optional,
@@ -72,6 +75,14 @@
7275
azure-core,
7376
azure-identity,
7477
azure-json,
78+
azure-xml,
79+
azure-storage-blob,
80+
azure-storage-common,
81+
azure-storage-internal-avro,
82+
com.microsoft.aad,
83+
com.microsoft.aad.msal4jextensions,
84+
com.microsoft.aad.msal4jextensions.persistence,
85+
jackson-dataformat-xml,
7586
guava,
7687
jsr305,
7788
reactive-streams,
@@ -174,6 +185,11 @@
174185
<groupId>com.microsoft.azure</groupId>
175186
<artifactId>azure-storage</artifactId>
176187
</dependency>
188+
<dependency>
189+
<groupId>com.azure</groupId>
190+
<artifactId>azure-storage-blob</artifactId>
191+
<version>12.27.1</version>
192+
</dependency>
177193
<dependency>
178194
<groupId>com.microsoft.azure</groupId>
179195
<artifactId>azure-keyvault-core</artifactId>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
2+
3+
import org.apache.jackrabbit.core.data.DataIdentifier;
4+
import org.apache.jackrabbit.core.data.DataRecord;
5+
import org.apache.jackrabbit.core.data.DataStoreException;
6+
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
7+
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUpload;
8+
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
9+
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadOptions;
10+
import org.apache.jackrabbit.oak.spi.blob.AbstractSharedBackend;
11+
import org.jetbrains.annotations.NotNull;
12+
13+
import java.net.URI;
14+
import java.util.Properties;
15+
16+
17+
public abstract class AbstractAzureBlobStoreBackend extends AbstractSharedBackend {
18+
19+
protected abstract DataRecordUpload initiateHttpUpload(long maxUploadSizeInBytes, int maxNumberOfURIs, @NotNull final DataRecordUploadOptions options);
20+
protected abstract DataRecord completeHttpUpload(@NotNull String uploadTokenStr) throws DataRecordUploadException, DataStoreException;
21+
protected abstract void setHttpDownloadURIExpirySeconds(int seconds);
22+
protected abstract void setHttpUploadURIExpirySeconds(int seconds);
23+
protected abstract void setHttpDownloadURICacheSize(int maxSize);
24+
protected abstract URI createHttpDownloadURI(@NotNull DataIdentifier identifier, @NotNull DataRecordDownloadOptions downloadOptions);
25+
public abstract void setProperties(final Properties properties);
26+
27+
}

oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobContainerProvider.java

Lines changed: 69 additions & 165 deletions
Large diffs are not rendered by default.

oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java

Lines changed: 284 additions & 411 deletions
Large diffs are not rendered by default.

oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStore.java

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.jackrabbit.core.data.DataIdentifier;
2626
import org.apache.jackrabbit.core.data.DataRecord;
2727
import org.apache.jackrabbit.core.data.DataStoreException;
28+
import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.v8.AzureBlobStoreBackendV8;
2829
import org.apache.jackrabbit.oak.plugins.blob.AbstractSharedCachingDataStore;
2930
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.ConfigurableDataRecordAccessProvider;
3031
import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordUploadException;
@@ -41,11 +42,18 @@ public class AzureDataStore extends AbstractSharedCachingDataStore implements Co
4142

4243
protected Properties properties;
4344

44-
private AzureBlobStoreBackend azureBlobStoreBackend;
45+
private AbstractAzureBlobStoreBackend azureBlobStoreBackend;
46+
47+
private final boolean useAzureSdkV12 = Boolean.getBoolean("blob.azure.v12.enabled");
4548

4649
@Override
4750
protected AbstractSharedBackend createBackend() {
48-
azureBlobStoreBackend = new AzureBlobStoreBackend();
51+
if(useAzureSdkV12) {
52+
azureBlobStoreBackend = new AzureBlobStoreBackend();
53+
} else {
54+
azureBlobStoreBackend = new AzureBlobStoreBackendV8();
55+
}
56+
4957
if (null != properties) {
5058
azureBlobStoreBackend.setProperties(properties);
5159
}

oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreService.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
2121

2222
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
23+
import org.jetbrains.annotations.NotNull;
2324
import org.osgi.service.component.annotations.Component;
2425
import org.osgi.service.component.annotations.ConfigurationPolicy;
2526
import org.osgi.service.component.annotations.Reference;
@@ -32,7 +33,7 @@ public class AzureDataStoreService extends AbstractAzureDataStoreService {
3233

3334
public static final String NAME = "org.apache.jackrabbit.oak.plugins.blob.datastore.AzureDataStore";
3435

35-
protected StatisticsProvider getStatisticsProvider(){
36+
protected @NotNull StatisticsProvider getStatisticsProvider(){
3637
return statisticsProvider;
3738
}
3839

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
2+
3+
public interface Constants {
4+
String META_DIR_NAME = "META";
5+
String META_KEY_PREFIX = META_DIR_NAME + "/";
6+
7+
String REF_KEY = "reference.key";
8+
String LAST_MODIFIED_KEY = "lastModified";
9+
10+
long BUFFERED_STREAM_THRESHOLD = 1024 * 1024;
11+
long MIN_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 10; // 10MB
12+
long MAX_MULTIPART_UPLOAD_PART_SIZE = 1024 * 1024 * 100; // 100MB
13+
long MAX_SINGLE_PUT_UPLOAD_SIZE = 1024 * 1024 * 256; // 256MB, Azure limit
14+
long MAX_BINARY_UPLOAD_SIZE = (long) Math.floor(1024L * 1024L * 1024L * 1024L * 4.75); // 4.75TB, Azure limit
15+
int MAX_ALLOWABLE_UPLOAD_URIS = 50000; // Azure limit
16+
int MAX_UNIQUE_RECORD_TRIES = 10;
17+
int DEFAULT_CONCURRENT_REQUEST_COUNT = 2;
18+
int MAX_CONCURRENT_REQUEST_COUNT = 50;
19+
}

oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/Utils.java

Lines changed: 61 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -1,130 +1,92 @@
1-
/*
2-
* Licensed to the Apache Software Foundation (ASF) under one
3-
* or more contributor license agreements. See the NOTICE file
4-
* distributed with this work for additional information
5-
* regarding copyright ownership. The ASF licenses this file
6-
* to you under the Apache License, Version 2.0 (the
7-
* "License"); you may not use this file except in compliance
8-
* with the License. You may obtain a copy of the License at
9-
*
10-
* http://www.apache.org/licenses/LICENSE-2.0
11-
*
12-
* Unless required by applicable law or agreed to in writing,
13-
* software distributed under the License is distributed on an
14-
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15-
* KIND, either express or implied. See the License for the
16-
* specific language governing permissions and limitations
17-
* under the License.
18-
*/
19-
201
package org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage;
212

22-
import java.io.File;
23-
import java.io.FileInputStream;
24-
import java.io.IOException;
25-
import java.io.InputStream;
26-
import java.net.InetSocketAddress;
27-
import java.net.Proxy;
28-
import java.net.SocketAddress;
29-
import java.net.URISyntaxException;
30-
import java.security.InvalidKeyException;
31-
import java.util.Properties;
32-
33-
import org.apache.jackrabbit.guava.common.base.Strings;
34-
import com.microsoft.azure.storage.CloudStorageAccount;
35-
import com.microsoft.azure.storage.OperationContext;
36-
import com.microsoft.azure.storage.RetryExponentialRetry;
37-
import com.microsoft.azure.storage.RetryNoRetry;
38-
import com.microsoft.azure.storage.RetryPolicy;
39-
import com.microsoft.azure.storage.StorageException;
40-
import com.microsoft.azure.storage.blob.BlobRequestOptions;
41-
import com.microsoft.azure.storage.blob.CloudBlobClient;
42-
import com.microsoft.azure.storage.blob.CloudBlobContainer;
3+
import com.azure.core.http.HttpClient;
4+
import com.azure.core.http.ProxyOptions;
5+
import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
6+
import com.azure.storage.blob.BlobContainerClient;
7+
import com.azure.storage.blob.BlobServiceClient;
8+
import com.azure.storage.blob.BlobServiceClientBuilder;
9+
import com.azure.storage.common.policy.RequestRetryOptions;
10+
import com.azure.storage.common.policy.RetryPolicyType;
11+
import com.google.common.base.Strings;
4312
import org.apache.commons.lang3.StringUtils;
4413
import org.apache.jackrabbit.core.data.DataStoreException;
4514
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
4615
import org.jetbrains.annotations.NotNull;
4716
import org.jetbrains.annotations.Nullable;
4817

49-
public final class Utils {
50-
51-
public static final String DEFAULT_CONFIG_FILE = "azure.properties";
18+
import java.io.File;
19+
import java.io.FileInputStream;
20+
import java.io.IOException;
21+
import java.io.InputStream;
22+
import java.net.InetSocketAddress;
23+
import java.time.Duration;
24+
import java.util.Properties;
5225

26+
public class Utils {
5327
public static final String DASH = "-";
28+
public static final String DEFAULT_CONFIG_FILE = "azure.properties";
5429

55-
/**
56-
* private constructor so that class cannot initialized from outside.
57-
*/
58-
private Utils() {
59-
}
60-
61-
/**
62-
* Create CloudBlobClient from properties.
63-
*
64-
* @param connectionString connectionString to configure @link {@link CloudBlobClient}
65-
* @return {@link CloudBlobClient}
66-
*/
67-
public static CloudBlobClient getBlobClient(@NotNull final String connectionString) throws URISyntaxException, InvalidKeyException {
68-
return getBlobClient(connectionString, null);
69-
}
30+
public Utils() {}
7031

71-
public static CloudBlobClient getBlobClient(@NotNull final String connectionString,
72-
@Nullable final BlobRequestOptions requestOptions) throws URISyntaxException, InvalidKeyException {
73-
CloudStorageAccount account = CloudStorageAccount.parse(connectionString);
74-
CloudBlobClient client = account.createCloudBlobClient();
75-
if (null != requestOptions) {
76-
client.setDefaultRequestOptions(requestOptions);
77-
}
78-
return client;
79-
}
32+
public static BlobContainerClient getBlobContainer(String accountName, String accountKey, @NotNull final String connectionString,
33+
@NotNull final String containerName,
34+
@Nullable final RequestRetryOptions retryOptions,
35+
boolean isProxyNeeded, final Properties properties) throws DataStoreException {
36+
BlobContainerClient blobContainerClient;
37+
try {
38+
if(isProxyNeeded) {
39+
String proxyHost = properties.getProperty(AzureConstants.PROXY_HOST);
40+
String proxyPort = properties.getProperty(AzureConstants.PROXY_PORT);
41+
42+
ProxyOptions proxyOptions = new ProxyOptions(ProxyOptions.Type.HTTP,
43+
new InetSocketAddress(proxyHost, Integer.parseInt(proxyPort)));
44+
45+
HttpClient httpClient = new NettyAsyncHttpClientBuilder()
46+
.proxy(proxyOptions)
47+
.build();
48+
49+
BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
50+
.connectionString(connectionString)
51+
.httpClient(httpClient)
52+
.retryOptions(retryOptions)
53+
.buildClient();
54+
blobContainerClient = blobServiceClient.getBlobContainerClient(containerName);
55+
return blobContainerClient;
56+
}
8057

81-
public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
82-
@NotNull final String containerName) throws DataStoreException {
83-
return getBlobContainer(connectionString, containerName, null);
84-
}
58+
BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
59+
.connectionString(connectionString)
60+
.retryOptions(retryOptions)
61+
.buildClient();
62+
blobContainerClient = blobServiceClient.getBlobContainerClient(containerName);
63+
return blobContainerClient;
8564

86-
public static CloudBlobContainer getBlobContainer(@NotNull final String connectionString,
87-
@NotNull final String containerName,
88-
@Nullable final BlobRequestOptions requestOptions) throws DataStoreException {
89-
try {
90-
CloudBlobClient client = (
91-
(null == requestOptions)
92-
? Utils.getBlobClient(connectionString)
93-
: Utils.getBlobClient(connectionString, requestOptions)
94-
);
95-
return client.getContainerReference(containerName);
96-
} catch (InvalidKeyException | URISyntaxException | StorageException e) {
65+
} catch (Exception e) {
9766
throw new DataStoreException(e);
9867
}
9968
}
10069

101-
public static void setProxyIfNeeded(final Properties properties) {
70+
public static boolean isProxyNeeded(final Properties properties) {
10271
String proxyHost = properties.getProperty(AzureConstants.PROXY_HOST);
10372
String proxyPort = properties.getProperty(AzureConstants.PROXY_PORT);
10473

105-
if (!Strings.isNullOrEmpty(proxyHost) &&
106-
Strings.isNullOrEmpty(proxyPort)) {
107-
int port = Integer.parseInt(proxyPort);
108-
SocketAddress proxyAddr = new InetSocketAddress(proxyHost, port);
109-
Proxy proxy = new Proxy(Proxy.Type.HTTP, proxyAddr);
110-
OperationContext.setDefaultProxy(proxy);
111-
}
74+
return !Strings.isNullOrEmpty(proxyHost) && Strings.isNullOrEmpty(proxyPort);
11275
}
11376

114-
public static RetryPolicy getRetryPolicy(final String maxRequestRetry) {
115-
int retries = PropertiesUtil.toInteger(maxRequestRetry, -1);
116-
if (retries < 0) {
77+
public static RequestRetryOptions getRetryOptions(final String maxRequestRetryCount) {
78+
int retries = PropertiesUtil.toInteger(maxRequestRetryCount, -1);
79+
if(retries < 0) {
11780
return null;
11881
}
82+
11983
if (retries == 0) {
120-
return new RetryNoRetry();
84+
return new RequestRetryOptions(RetryPolicyType.FIXED, 1, 1, null, null, null);
12185
}
122-
return new RetryExponentialRetry(RetryPolicy.DEFAULT_CLIENT_BACKOFF, retries);
86+
return new RequestRetryOptions(RetryPolicyType.EXPONENTIAL, retries, (int) Duration.ofSeconds(3).toMillis(), null, null, null);
12387
}
12488

125-
12689
public static String getConnectionStringFromProperties(Properties properties) {
127-
12890
String sasUri = properties.getProperty(AzureConstants.AZURE_SAS, "");
12991
String blobEndpoint = properties.getProperty(AzureConstants.AZURE_BLOB_ENDPOINT, "");
13092
String connectionString = properties.getProperty(AzureConstants.AZURE_CONNECTION_STRING, "");
@@ -141,7 +103,7 @@ public static String getConnectionStringFromProperties(Properties properties) {
141103

142104
return getConnectionString(
143105
accountName,
144-
accountKey,
106+
accountKey,
145107
blobEndpoint);
146108
}
147109

@@ -153,15 +115,11 @@ public static String getConnectionStringForSas(String sasUri, String blobEndpoin
153115
}
154116
}
155117

156-
public static String getConnectionString(final String accountName, final String accountKey) {
157-
return getConnectionString(accountName, accountKey, null);
158-
}
159-
160118
public static String getConnectionString(final String accountName, final String accountKey, String blobEndpoint) {
161119
StringBuilder connString = new StringBuilder("DefaultEndpointsProtocol=https");
162120
connString.append(";AccountName=").append(accountName);
163121
connString.append(";AccountKey=").append(accountKey);
164-
122+
165123
if (!Strings.isNullOrEmpty(blobEndpoint)) {
166124
connString.append(";BlobEndpoint=").append(blobEndpoint);
167125
}

0 commit comments

Comments
 (0)