Description
Describe the bug
This issue seems to be with presigner. We want to have presigned URI that will be used by end client to upload objects and complete them the "completeMultipartUpload" request throws exception. We have also tried by changing content type from xml to octet-stream, but still the issue does not resolve. We cannot use cURL from command line with our requirements.
Regression Issue
- Select this option if this issue appears to be a regression.
Expected Behavior
- CompletedMultipartUpload give http 200 not 403 with code 200.
- The signatures used to sign match as SDK's builder adds them.
Current Behavior
[ Response body: ] SignatureDoesNotMatchThe request signature we calculated does not match the signature you provided. Check your key and signing method.AKIA3DW4R6EQKAVGXSPXAWS4-HMAC-SHA256
20241029T101952Z
20241029/ap-south-1/s3/aws4_request
Reproduction Steps
We get the same error. We have refactored the code according to our requirements.
package com.cio.storage.aws.examples;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.dataformat.xml.XmlMapper;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.http.SdkHttpMethod;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
import software.amazon.awssdk.services.s3.model.CompletedPart;
import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.CompleteMultipartUploadPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.CreateMultipartUploadPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedCompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedCreateMultipartUploadRequest;
import software.amazon.awssdk.services.s3.presigner.model.PresignedUploadPartRequest;
import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest;
import software.amazon.awssdk.utils.IoUtils;
/**
- Actual Example taken from
- {@link : https://github.com/aws/aws-sdk-java-v2/issues/4196}
*/
public class TestMPUWithPresignedURLsGH
{
final static String CREATE_MPU_UR_CT = "application/octet-stream";
final static String PRESIGNED_MPU_UR_CT = "application/octet-stream";
private final XmlMapper xmlMapper = new XmlMapper();
public static void main(final String[] args)
{
try
{
new TestMPUWithPresignedURLsGH().test();
} catch (Exception ex)
{
ex.printStackTrace();
}
}
private void test() throws IOException, XMLStreamException
{
// FILL THE INFORMATION HERE
final String accessKeyId = "";
final String secretAccessKey = "";
final Region region = ;
final String bucket = "";
final String key = "";
this.putObjectMPU(region, accessKeyId, secretAccessKey, bucket, key);
}
private void putObjectMPU(final Region region, final String accessKeyId, final String secretAccessKey, final String bucket, final String key)
throws IOException, XMLStreamException
{
// Create S3 presigner
final AwsCredentialsProvider awsCredentialsProvider = () -> AwsBasicCredentials.create(accessKeyId, secretAccessKey);
final S3Configuration s3Configuration = S3Configuration.builder().pathStyleAccessEnabled(false).build();
final S3Presigner s3Presigner = S3Presigner.builder().credentialsProvider(awsCredentialsProvider).region(region)
.serviceConfiguration(s3Configuration).build();
// -->
// Create MPU
final CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder().bucket(bucket).key(key)
.contentType(TestMPUWithPresignedURLsGH.CREATE_MPU_UR_CT).build();
final CreateMultipartUploadPresignRequest createMultipartUploadPresignRequest = CreateMultipartUploadPresignRequest.builder()
.signatureDuration(Duration.ofHours(1)).createMultipartUploadRequest(createMultipartUploadRequest).build();
final PresignedCreateMultipartUploadRequest presignedCreateMultipartUploadRequest = s3Presigner
.presignCreateMultipartUpload(createMultipartUploadPresignRequest);
final URL presignedCreateMultipartUploadRequestURL = presignedCreateMultipartUploadRequest.url();
System.out.println("Create MPU URL: " + presignedCreateMultipartUploadRequestURL);
final HttpURLResponse presignedCreateMultipartUploadRequestResponse = this.makeRequest(SdkHttpMethod.POST,
presignedCreateMultipartUploadRequestURL, PRESIGNED_MPU_UR_CT, /* data */ null);
// final HttpURLResponse presignedCreateMultipartUploadRequestResponse = this.makeRequest(SdkHttpMethod.POST,
// presignedCreateMultipartUploadRequestURL, PRESIGNED_MPU_UR_CT, /* data */
// null);
final JsonNode presignedCreateMultipartUploadRequestResponseXml = this.fromXml(presignedCreateMultipartUploadRequestResponse.getBody());
final String multipartUploadId = presignedCreateMultipartUploadRequestResponseXml.get("UploadId").asText();
System.out.println("MPU ID: " + multipartUploadId);
// -->
// MPU Part
final int partNumber = 1;
final String partContent = "AAA" + this.createLongString(1024 * 1024 * 5);
final byte[] partContentBytes = partContent.getBytes(StandardCharsets.UTF_8);
System.out.printf("UploadPartRequest #%d: bucket=%s key=%s upload-id=%s%n", partNumber, bucket, key, multipartUploadId);
final UploadPartRequest uploadPartRequest = UploadPartRequest.builder().bucket(bucket).key(key).uploadId(multipartUploadId)
.partNumber(partNumber).build();
final UploadPartPresignRequest uploadPartPresignRequest = UploadPartPresignRequest.builder().signatureDuration(Duration.ofHours(1))
.uploadPartRequest(uploadPartRequest).build();
final PresignedUploadPartRequest presignedUploadPartRequest = s3Presigner.presignUploadPart(uploadPartPresignRequest);
final URL presignedUploadPartRequestURL = presignedUploadPartRequest.url();
System.out.printf("UploadPart #%d URL: %s%n", partNumber, presignedUploadPartRequestURL);
final HttpURLResponse presignedUploadPartRequest1Response = this.makeRequest(SdkHttpMethod.PUT, presignedUploadPartRequestURL,
/* contentType */ null, partContentBytes);
final String eTag = presignedUploadPartRequest1Response.getHeaders().get("ETag").get(0);
System.out.printf("UploadPart #%d ETag: %s%n", partNumber, eTag);
final CompletedPart completedPart = CompletedPart.builder().partNumber(partNumber).eTag(eTag).build();
final String eTagPart = completedPart.eTag();
// -->
// Commit MPU
final CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder().parts(completedPart).build();
System.out.printf("CompleteMultipartUploadRequest: bucket=%s key=%s multipart-id=%s%n", bucket, key, multipartUploadId);
final CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder().bucket(bucket).key(key)
.uploadId(multipartUploadId).multipartUpload(completedMultipartUpload).build();
final CompleteMultipartUploadPresignRequest completeMultipartUploadPresignRequest = CompleteMultipartUploadPresignRequest.builder()
.signatureDuration(Duration.ofHours(1)).completeMultipartUploadRequest(completeMultipartUploadRequest).build();
final PresignedCompleteMultipartUploadRequest presignedCompleteMultipartUploadRequest = s3Presigner
.presignCompleteMultipartUpload(completeMultipartUploadPresignRequest);
final URL presignedCompleteMultipartUploadRequestURL = presignedCompleteMultipartUploadRequest.url();
System.out.printf("CompleteMultipartUpload URL: %s%n", presignedCompleteMultipartUploadRequestURL);
final Map<Integer, String> parts = Collections.singletonMap(partNumber, eTagPart);
final String completeMultipartRequestPayload = this.buildCompleteMultipartRequestPayload(parts);
final byte[] completeMultipartRequestPayloadBytes = completeMultipartRequestPayload.getBytes(StandardCharsets.UTF_8);
System.out.println("Request payload: " + new String(completeMultipartRequestPayloadBytes, StandardCharsets.UTF_8));
String contentTypeComMPUResponse = "application/x-www-form-urlencoded";
// String contentTypeComMPUResponse = "application/octet-stream";
final HttpURLResponse completeMultipartUploadResponse = this.makeRequest(SdkHttpMethod.POST, presignedCompleteMultipartUploadRequestURL,
/* contentType */ contentTypeComMPUResponse, completeMultipartRequestPayloadBytes);
System.out.println("Status Code: " + completeMultipartUploadResponse.getStatusCode());
System.out.println("Response headers: " + completeMultipartUploadResponse.getHeaders());
System.out.println("\n [ Response body: ] " + completeMultipartUploadResponse.getBody());
// -->
}
private String buildCompleteMultipartRequestPayload(final Map<Integer, String> parts) throws XMLStreamException
{
final StringWriter stringWriter = new StringWriter();
final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newFactory();
final XMLStreamWriter xmlWriter = xmlOutputFactory.createXMLStreamWriter(stringWriter);
xmlWriter.writeStartDocument();
xmlWriter.writeStartElement("CompleteMultipartUpload");
xmlWriter.writeDefaultNamespace("http://s3.amazonaws.com/doc/2006-03-01/");
for (final Map.Entry<Integer, String> part : parts.entrySet())
{
xmlWriter.writeStartElement("Part");
xmlWriter.writeStartElement("ETag");
xmlWriter.writeCharacters(part.getValue());
xmlWriter.writeEndElement();
xmlWriter.writeStartElement("PartNumber");
xmlWriter.writeCharacters(part.getKey().toString());
xmlWriter.writeEndElement();
xmlWriter.writeEndElement();
}
xmlWriter.writeEndElement();
xmlWriter.writeEndDocument();
return stringWriter.toString().replace("<?xml version='1.0' encoding='UTF-8'?>", "");
}
private String createLongString(final int length)
{
final char[] randomChars = new char[length];
Arrays.fill(randomChars, 'z');
return new String(randomChars);
}
private JsonNode fromXml(final String xml) throws JsonProcessingException
{
return this.xmlMapper.readTree(xml);
}
private HttpURLResponse makeRequest(final SdkHttpMethod httpMethod, final URL url, final String contentType, final byte[] data) throws IOException
{
System.out.println("Making request to: " + url.toString());
final HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection();
httpURLConnection.setRequestMethod(httpMethod.name());
httpURLConnection.setDefaultUseCaches(false);
if (contentType != null)
{
httpURLConnection.setRequestProperty("Content-Type", contentType);
}
if (data != null)
{
httpURLConnection.setDoOutput(true);
try (final InputStream requestPayloadInputStream = new ByteArrayInputStream(data);
final OutputStream httpOutputStream = httpURLConnection.getOutputStream())
{
IoUtils.copy(requestPayloadInputStream, httpOutputStream);
}
}
return new HttpURLResponse(httpURLConnection);
}
private static class HttpURLResponse
{
private final int statusCode;
private final Map<String, List<String>> headers;
private final String body;
public HttpURLResponse(final HttpURLConnection httpURLConnection) throws IOException
{
this.statusCode = httpURLConnection.getResponseCode();
this.headers = httpURLConnection.getHeaderFields();
if (this.statusCode == HttpURLConnection.HTTP_OK)
{
try (final InputStream in = httpURLConnection.getInputStream(); final ByteArrayOutputStream out = new ByteArrayOutputStream())
{
IoUtils.copy(in, out);
final byte[] bodyBytes = out.toByteArray();
this.body = new String(bodyBytes, StandardCharsets.UTF_8);
}
} else
{
try (final InputStream in = httpURLConnection.getErrorStream(); final ByteArrayOutputStream out = new ByteArrayOutputStream())
{
IoUtils.copy(in, out);
final byte[] bodyBytes = out.toByteArray();
this.body = new String(bodyBytes, StandardCharsets.UTF_8);
}
}
}
public int getStatusCode()
{
return this.statusCode;
}
public Map<String, List<String>> getHeaders()
{
return this.headers;
}
public String getBody()
{
return this.body;
}
}
}
Possible Solution
No response
Additional Information/Context
No response
AWS Java SDK version used
2.29.1, 2.27.12 and some others
JDK version used
1.8
Operating System and version
Windows 10 and Ubuntu