Skip to content

Commit 160fa9a

Browse files
ShreyasSinhaBenWhitehead
ShreyasSinha
and
BenWhitehead
committed
chore: mpu client preview merge train
BEGIN_NESTED_COMMIT BEGIN_COMMIT_OVERRIDE feat: add preview MultipartUploadClient#createMultipartUpload #3356 END_COMMIT_OVERRIDE BEGIN_COMMIT_OVERRIDE feat: add preview MultipartUploadClient#listParts #3359 END_COMMIT_OVERRIDE BEGIN_COMMIT_OVERRIDE feat: add preview MultipartUploadClient#abortMultipartUpload #3361 END_COMMIT_OVERRIDE BEGIN_COMMIT_OVERRIDE feat: add preview MultipartUploadClient#uploadPart #3375 END_COMMIT_OVERRIDE BEGIN_COMMIT_OVERRIDE feat: add preview MultipartUploadClient#completeMultipartUpload #3372 END_COMMIT_OVERRIDE BEGIN_COMMIT_OVERRIDE feat: add preview MultipartUploadSettings END_COMMIT_OVERRIDE BEGIN_COMMIT_OVERRIDE END_COMMIT_OVERRIDE END_NESTED_COMMIT Other changes: 1. chore: refactor retrier creation from HttpStorageOptions to StorageOptions #3350 2. chore: refactorings for CreateMultipartUpload #3364 3. chore: fix xml parsing of StringEnumValue's so that the enum contract is not broken #3377 4. chore: add PredefinedAcl#xmlEntry Co-authored-by: BenWhitehead <[email protected]>
1 parent 87c9c81 commit 160fa9a

32 files changed

+5218
-17
lines changed

‎google-cloud-storage/pom.xml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,22 @@
1919
<pubsub-proto.version>1.125.0</pubsub-proto.version>
2020
</properties>
2121
<dependencies>
22+
<dependency>
23+
<groupId>com.fasterxml.jackson.dataformat</groupId>
24+
<artifactId>jackson-dataformat-xml</artifactId>
25+
</dependency>
26+
<dependency>
27+
<groupId>com.fasterxml.jackson.datatype</groupId>
28+
<artifactId>jackson-datatype-jsr310</artifactId>
29+
</dependency>
30+
<dependency>
31+
<groupId>com.fasterxml.jackson.core</groupId>
32+
<artifactId>jackson-databind</artifactId>
33+
</dependency>
34+
<dependency>
35+
<groupId>com.fasterxml.jackson.core</groupId>
36+
<artifactId>jackson-annotations</artifactId>
37+
</dependency>
2238
<dependency>
2339
<groupId>com.google.guava</groupId>
2440
<artifactId>guava</artifactId>

‎google-cloud-storage/src/main/java/com/google/cloud/storage/ChecksumResponseParser.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.api.client.http.HttpResponse;
20+
import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadResponse;
21+
import com.google.cloud.storage.multipartupload.model.UploadPartResponse;
22+
import java.io.IOException;
23+
import java.util.Arrays;
24+
import java.util.Collections;
25+
import java.util.Map;
26+
import java.util.Optional;
27+
import java.util.stream.Collectors;
28+
29+
/** A utility class to parse {@link HttpResponse} and create a {@link UploadPartResponse}. */
30+
final class ChecksumResponseParser {
31+
32+
private ChecksumResponseParser() {}
33+
34+
static UploadPartResponse parseUploadResponse(HttpResponse response) {
35+
String eTag = response.getHeaders().getETag();
36+
Map<String, String> hashes = extractHashesFromHeader(response);
37+
return UploadPartResponse.builder().eTag(eTag).md5(hashes.get("md5")).build();
38+
}
39+
40+
static CompleteMultipartUploadResponse parseCompleteResponse(HttpResponse response)
41+
throws IOException {
42+
Map<String, String> hashes = extractHashesFromHeader(response);
43+
CompleteMultipartUploadResponse completeMpuResponse =
44+
response.parseAs(CompleteMultipartUploadResponse.class);
45+
return CompleteMultipartUploadResponse.builder()
46+
.location(completeMpuResponse.location())
47+
.bucket(completeMpuResponse.bucket())
48+
.key(completeMpuResponse.key())
49+
.etag(completeMpuResponse.etag())
50+
.crc32c(hashes.get("crc32c"))
51+
.build();
52+
}
53+
54+
static Map<String, String> extractHashesFromHeader(HttpResponse response) {
55+
return Optional.ofNullable(response.getHeaders().getFirstHeaderStringValue("x-goog-hash"))
56+
.map(
57+
h ->
58+
Arrays.stream(h.split(","))
59+
.map(s -> s.trim().split("=", 2))
60+
.filter(a -> a.length == 2)
61+
.filter(a -> "crc32c".equalsIgnoreCase(a[0]) || "md5".equalsIgnoreCase(a[0]))
62+
.collect(Collectors.toMap(a -> a[0].toLowerCase(), a -> a[1], (v1, v2) -> v1)))
63+
.orElse(Collections.emptyMap());
64+
}
65+
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/HttpRetryAlgorithmManager.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,10 @@ ResultRetryAlgorithm<?> idempotent() {
5050
return retryStrategy.getIdempotentHandler();
5151
}
5252

53+
ResultRetryAlgorithm<?> nonIdempotent() {
54+
return retryStrategy.getNonidempotentHandler();
55+
}
56+
5357
public ResultRetryAlgorithm<?> getForBucketAclCreate(
5458
BucketAccessControl pb, Map<StorageRpc.Option, ?> optionsMap) {
5559
return retryStrategy.getNonidempotentHandler();

‎google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import com.google.cloud.http.HttpTransportOptions;
3333
import com.google.cloud.spi.ServiceRpcFactory;
3434
import com.google.cloud.storage.BlobWriteSessionConfig.WriterFactory;
35-
import com.google.cloud.storage.Retrying.DefaultRetrier;
3635
import com.google.cloud.storage.Retrying.HttpRetrier;
3736
import com.google.cloud.storage.Retrying.RetryingDependencies;
3837
import com.google.cloud.storage.Storage.BlobWriteOption;
@@ -409,13 +408,7 @@ public Storage create(StorageOptions options) {
409408
WriterFactory factory = blobWriteSessionConfig.createFactory(clock);
410409
StorageImpl storage =
411410
new StorageImpl(
412-
httpStorageOptions,
413-
factory,
414-
new HttpRetrier(
415-
new DefaultRetrier(
416-
OtelStorageDecorator.retryContextDecorator(otel),
417-
RetryingDependencies.simple(
418-
options.getClock(), options.getRetrySettings()))));
411+
httpStorageOptions, factory, new HttpRetrier(options.createRetrier()));
419412
return OtelStorageDecorator.decorate(storage, otel, Transport.HTTP);
420413
} catch (IOException e) {
421414
throw new IllegalStateException(

‎google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClient.java‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage;
18+
19+
import com.google.api.core.BetaApi;
20+
import com.google.api.core.InternalExtensionOnly;
21+
import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadRequest;
22+
import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadResponse;
23+
import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadRequest;
24+
import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadResponse;
25+
import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest;
26+
import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse;
27+
import com.google.cloud.storage.multipartupload.model.ListPartsRequest;
28+
import com.google.cloud.storage.multipartupload.model.ListPartsResponse;
29+
import com.google.cloud.storage.multipartupload.model.UploadPartRequest;
30+
import com.google.cloud.storage.multipartupload.model.UploadPartResponse;
31+
import java.net.URI;
32+
33+
/**
34+
* A client for interacting with Google Cloud Storage's Multipart Upload API.
35+
*
36+
* <p>This class is for internal use only and is not intended for public consumption. It provides a
37+
* low-level interface for creating and managing multipart uploads.
38+
*
39+
* @see <a href="https://cloud.google.com/storage/docs/multipart-uploads">Multipart Uploads</a>
40+
* @since 2.60.0 This new api is in preview and is subject to breaking changes.
41+
*/
42+
@BetaApi
43+
@InternalExtensionOnly
44+
public abstract class MultipartUploadClient {
45+
46+
MultipartUploadClient() {}
47+
48+
/**
49+
* Creates a new multipart upload.
50+
*
51+
* @param request The request object containing the details for creating the multipart upload.
52+
* @return A {@link CreateMultipartUploadResponse} object containing the upload ID.
53+
* @since 2.60.0 This new api is in preview and is subject to breaking changes.
54+
*/
55+
@BetaApi
56+
public abstract CreateMultipartUploadResponse createMultipartUpload(
57+
CreateMultipartUploadRequest request);
58+
59+
/**
60+
* Lists the parts that have been uploaded for a specific multipart upload.
61+
*
62+
* @param listPartsRequest The request object containing the details for listing the parts.
63+
* @return A {@link ListPartsResponse} object containing the list of parts.
64+
* @since 2.60.0 This new api is in preview and is subject to breaking changes.
65+
*/
66+
@BetaApi
67+
public abstract ListPartsResponse listParts(ListPartsRequest listPartsRequest);
68+
69+
/**
70+
* Aborts a multipart upload.
71+
*
72+
* @param request The request object containing the details for aborting the multipart upload.
73+
* @return An {@link AbortMultipartUploadResponse} object.
74+
* @since 2.60.0 This new api is in preview and is subject to breaking changes.
75+
*/
76+
@BetaApi
77+
public abstract AbortMultipartUploadResponse abortMultipartUpload(
78+
AbortMultipartUploadRequest request);
79+
80+
/**
81+
* Completes a multipart upload.
82+
*
83+
* @param request The request object containing the details for completing the multipart upload.
84+
* @return A {@link CompleteMultipartUploadResponse} object containing information about the
85+
* completed upload.
86+
* @since 2.60.0 This new api is in preview and is subject to breaking changes.
87+
*/
88+
@BetaApi
89+
public abstract CompleteMultipartUploadResponse completeMultipartUpload(
90+
CompleteMultipartUploadRequest request);
91+
92+
/**
93+
* Uploads a part in a multipart upload.
94+
*
95+
* @param request The request object containing the details for uploading the part.
96+
* @param requestBody The content of the part to upload.
97+
* @return An {@link UploadPartResponse} object containing the ETag of the uploaded part.
98+
* @since 2.60.0 This new api is in preview and is subject to breaking changes.
99+
*/
100+
@BetaApi
101+
public abstract UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requestBody);
102+
103+
/**
104+
* Creates a new instance of {@link MultipartUploadClient}.
105+
*
106+
* @param config The configuration for the client.
107+
* @return A new {@link MultipartUploadClient} instance.
108+
* @since 2.60.0 This new api is in preview and is subject to breaking changes.
109+
*/
110+
@BetaApi
111+
public static MultipartUploadClient create(MultipartUploadSettings config) {
112+
HttpStorageOptions options = config.getOptions();
113+
return new MultipartUploadClientImpl(
114+
URI.create(options.getHost()),
115+
options.createRetrier(),
116+
MultipartUploadHttpRequestManager.createFrom(options),
117+
options.getRetryAlgorithmManager());
118+
}
119+
}

‎google-cloud-storage/src/main/java/com/google/cloud/storage/MultipartUploadClientImpl.java‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.storage;
17+
18+
import com.google.cloud.storage.Conversions.Decoder;
19+
import com.google.cloud.storage.Retrying.Retrier;
20+
import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadRequest;
21+
import com.google.cloud.storage.multipartupload.model.AbortMultipartUploadResponse;
22+
import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadRequest;
23+
import com.google.cloud.storage.multipartupload.model.CompleteMultipartUploadResponse;
24+
import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadRequest;
25+
import com.google.cloud.storage.multipartupload.model.CreateMultipartUploadResponse;
26+
import com.google.cloud.storage.multipartupload.model.ListPartsRequest;
27+
import com.google.cloud.storage.multipartupload.model.ListPartsResponse;
28+
import com.google.cloud.storage.multipartupload.model.UploadPartRequest;
29+
import com.google.cloud.storage.multipartupload.model.UploadPartResponse;
30+
import java.net.URI;
31+
import java.util.concurrent.atomic.AtomicBoolean;
32+
33+
/**
34+
* This class is an implementation of {@link MultipartUploadClient} that uses the Google Cloud
35+
* Storage XML API to perform multipart uploads.
36+
*/
37+
final class MultipartUploadClientImpl extends MultipartUploadClient {
38+
39+
private final MultipartUploadHttpRequestManager httpRequestManager;
40+
private final Retrier retrier;
41+
private final URI uri;
42+
private final HttpRetryAlgorithmManager retryAlgorithmManager;
43+
44+
MultipartUploadClientImpl(
45+
URI uri,
46+
Retrier retrier,
47+
MultipartUploadHttpRequestManager multipartUploadHttpRequestManager,
48+
HttpRetryAlgorithmManager retryAlgorithmManager) {
49+
this.httpRequestManager = multipartUploadHttpRequestManager;
50+
this.retrier = retrier;
51+
this.uri = uri;
52+
this.retryAlgorithmManager = retryAlgorithmManager;
53+
}
54+
55+
@Override
56+
public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUploadRequest request) {
57+
return retrier.run(
58+
retryAlgorithmManager.nonIdempotent(),
59+
() -> httpRequestManager.sendCreateMultipartUploadRequest(uri, request),
60+
Decoder.identity());
61+
}
62+
63+
@Override
64+
public ListPartsResponse listParts(ListPartsRequest request) {
65+
66+
return retrier.run(
67+
retryAlgorithmManager.idempotent(),
68+
() -> httpRequestManager.sendListPartsRequest(uri, request),
69+
Decoder.identity());
70+
}
71+
72+
@Override
73+
public AbortMultipartUploadResponse abortMultipartUpload(AbortMultipartUploadRequest request) {
74+
75+
return retrier.run(
76+
retryAlgorithmManager.idempotent(),
77+
() -> httpRequestManager.sendAbortMultipartUploadRequest(uri, request),
78+
Decoder.identity());
79+
}
80+
81+
@Override
82+
public CompleteMultipartUploadResponse completeMultipartUpload(
83+
CompleteMultipartUploadRequest request) {
84+
return retrier.run(
85+
retryAlgorithmManager.idempotent(),
86+
() -> httpRequestManager.sendCompleteMultipartUploadRequest(uri, request),
87+
Decoder.identity());
88+
}
89+
90+
@Override
91+
public UploadPartResponse uploadPart(UploadPartRequest request, RequestBody requestBody) {
92+
AtomicBoolean dirty = new AtomicBoolean(false);
93+
return retrier.run(
94+
retryAlgorithmManager.idempotent(),
95+
() -> {
96+
if (dirty.getAndSet(true)) {
97+
requestBody.getContent().rewindTo(0);
98+
}
99+
return httpRequestManager.sendUploadPartRequest(uri, request, requestBody.getContent());
100+
},
101+
Decoder.identity());
102+
}
103+
}

0 commit comments

Comments
 (0)