Skip to content

Commit 12f6265

Browse files
authored
GH-45304: [C++][S3] Workaround compatibility issue between AWS SDK and MinIO (#45310)
### Rationale for this change Some AWS SDK versions have faulty chunked encoding when the body is 0 bytes: aws/aws-sdk-cpp#3259 ### What changes are included in this PR? Work around faulty chunked encoding implementation by only setting a body stream if non-empty. ### Are these changes tested? Locally for now, but will be picked by CI (and conda-forge) at some point. ### Are there any user-facing changes? No. * GitHub Issue: #45304 Authored-by: Antoine Pitrou <[email protected]> Signed-off-by: Antoine Pitrou <[email protected]>
1 parent 3a3f2ec commit 12f6265

File tree

2 files changed

+17
-13
lines changed

2 files changed

+17
-13
lines changed

cpp/src/arrow/filesystem/s3fs.cc

+17-12
Original file line numberDiff line numberDiff line change
@@ -1983,27 +1983,33 @@ class ObjectOutputStream final : public io::OutputStream {
19831983
const void* data, int64_t nbytes, std::shared_ptr<Buffer> owned_buffer = nullptr) {
19841984
req.SetBucket(ToAwsString(path_.bucket));
19851985
req.SetKey(ToAwsString(path_.key));
1986-
req.SetBody(std::make_shared<StringViewStream>(data, nbytes));
19871986
req.SetContentLength(nbytes);
19881987
RETURN_NOT_OK(SetSSECustomerKey(&req, sse_customer_key_));
19891988

19901989
if (!background_writes_) {
1991-
req.SetBody(std::make_shared<StringViewStream>(data, nbytes));
1990+
// GH-45304: avoid setting a body stream if length is 0.
1991+
// This workaround can be removed once we require AWS SDK 1.11.489 or later.
1992+
if (nbytes != 0) {
1993+
req.SetBody(std::make_shared<StringViewStream>(data, nbytes));
1994+
}
19921995

19931996
ARROW_ASSIGN_OR_RAISE(auto outcome, TriggerUploadRequest(req, holder_));
19941997

19951998
RETURN_NOT_OK(sync_result_callback(req, upload_state_, part_number_, outcome));
19961999
} else {
1997-
// If the data isn't owned, make an immutable copy for the lifetime of the closure
1998-
if (owned_buffer == nullptr) {
1999-
ARROW_ASSIGN_OR_RAISE(owned_buffer, AllocateBuffer(nbytes, io_context_.pool()));
2000-
memcpy(owned_buffer->mutable_data(), data, nbytes);
2001-
} else {
2002-
DCHECK_EQ(data, owned_buffer->data());
2003-
DCHECK_EQ(nbytes, owned_buffer->size());
2000+
// (GH-45304: avoid setting a body stream if length is 0, see above)
2001+
if (nbytes != 0) {
2002+
// If the data isn't owned, make an immutable copy for the lifetime of the closure
2003+
if (owned_buffer == nullptr) {
2004+
ARROW_ASSIGN_OR_RAISE(owned_buffer, AllocateBuffer(nbytes, io_context_.pool()));
2005+
memcpy(owned_buffer->mutable_data(), data, nbytes);
2006+
} else {
2007+
DCHECK_EQ(data, owned_buffer->data());
2008+
DCHECK_EQ(nbytes, owned_buffer->size());
2009+
}
2010+
req.SetBody(std::make_shared<StringViewStream>(owned_buffer->data(),
2011+
owned_buffer->size()));
20042012
}
2005-
req.SetBody(
2006-
std::make_shared<StringViewStream>(owned_buffer->data(), owned_buffer->size()));
20072013

20082014
{
20092015
std::unique_lock<std::mutex> lock(upload_state_->mutex);
@@ -2345,7 +2351,6 @@ class S3FileSystem::Impl : public std::enable_shared_from_this<S3FileSystem::Imp
23452351
req.SetBucket(ToAwsString(bucket));
23462352
req.SetKey(ToAwsString(key));
23472353
req.SetContentType(kAwsDirectoryContentType);
2348-
req.SetBody(std::make_shared<std::stringstream>(""));
23492354
return OutcomeToStatus(
23502355
std::forward_as_tuple("When creating key '", key, "' in bucket '", bucket, "': "),
23512356
"PutObject", client_lock.Move()->PutObject(req));

cpp/src/arrow/filesystem/s3fs_test.cc

-1
Original file line numberDiff line numberDiff line change
@@ -526,7 +526,6 @@ class TestS3FS : public S3TestMixin {
526526
Aws::S3::Model::PutObjectRequest req;
527527
req.SetBucket(ToAwsString("bucket"));
528528
req.SetKey(ToAwsString("emptydir/"));
529-
req.SetBody(std::make_shared<std::stringstream>(""));
530529
RETURN_NOT_OK(OutcomeToStatus("PutObject", client_->PutObject(req)));
531530
// NOTE: no need to create intermediate "directories" somedir/ and
532531
// somedir/subdir/

0 commit comments

Comments
 (0)