From 9a9874279da5c152e546bed69fc808d2537be68e Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Sun, 16 Jun 2024 22:47:31 -0700 Subject: [PATCH 1/3] Request Id in Errors Signed-off-by: Haytham Abuelfutuh --- s3fs/errors.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/s3fs/errors.py b/s3fs/errors.py index f5350670..9f013bc5 100644 --- a/s3fs/errors.py +++ b/s3fs/errors.py @@ -140,15 +140,26 @@ def translate_boto_error(error, message=None, set_cause=True, *args, **kwargs): if error_response is None: # non-http error, or response is None: return error + + request_id = '' + if 'ResponseMetadata' in error_response and 'RequestId' in error_response['ResponseMetadata']: + request_id = error_response['ResponseMetadata']['RequestId'] + print(f'Request ID: {request_id}') + http_code = '' + if 'ResponseMetadata' in error_response and 'HTTPStatusCode' in error_response['ResponseMetadata']: + http_code = error_response['ResponseMetadata']['HTTPStatusCode'] + print(f"Http code: {http_code}") code = error_response["Error"].get("Code") constructor = ERROR_CODE_TO_EXCEPTION.get(code) if constructor: if not message: message = error_response["Error"].get("Message", str(error)) + message = f'{message}. Request ID: {request_id}, HTTP code: {http_code}' custom_exc = constructor(message, *args, **kwargs) else: # No match found, wrap this in an IOError with the appropriate message. - custom_exc = IOError(errno.EIO, message or str(error), *args) + message = f'{message or str(error)}. Request ID: {request_id}, HTTP code: {http_code}' + custom_exc = IOError(errno.EIO, message, *args) if set_cause: custom_exc.__cause__ = error From 23197da4d09320f48d85b4aae883cea4c3b97097 Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Mon, 17 Jun 2024 09:34:55 -0700 Subject: [PATCH 2/3] Cleanup debug statements Signed-off-by: Haytham Abuelfutuh --- s3fs/errors.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/s3fs/errors.py b/s3fs/errors.py index 9f013bc5..25478772 100644 --- a/s3fs/errors.py +++ b/s3fs/errors.py @@ -141,24 +141,28 @@ def translate_boto_error(error, message=None, set_cause=True, *args, **kwargs): # non-http error, or response is None: return error + # AWS s3 as well as moto (CI Backend) respond with ResponseMetadata that contains RequestId as well as + # HTTPStatusCode. As of 6/17/2024, minio doesn't seem to add RequestId to the response. request_id = '' if 'ResponseMetadata' in error_response and 'RequestId' in error_response['ResponseMetadata']: request_id = error_response['ResponseMetadata']['RequestId'] - print(f'Request ID: {request_id}') http_code = '' if 'ResponseMetadata' in error_response and 'HTTPStatusCode' in error_response['ResponseMetadata']: http_code = error_response['ResponseMetadata']['HTTPStatusCode'] - print(f"Http code: {http_code}") + code = error_response["Error"].get("Code") constructor = ERROR_CODE_TO_EXCEPTION.get(code) if constructor: if not message: message = error_response["Error"].get("Message", str(error)) - message = f'{message}. Request ID: {request_id}, HTTP code: {http_code}' + message = f'{message}, Request ID: {request_id}' if request_id else message + message = f'{message}, HTTP Status code: {http_code}' if http_code else message custom_exc = constructor(message, *args, **kwargs) else: # No match found, wrap this in an IOError with the appropriate message. - message = f'{message or str(error)}. Request ID: {request_id}, HTTP code: {http_code}' + message = message or str(error) + message = f'{message}, Request ID: {request_id}' if request_id else message + message = f'{message}, HTTP Status code: {http_code}' if http_code else message custom_exc = IOError(errno.EIO, message, *args) if set_cause: From eac2d4ce275a47000c858f367ce1b1924f54ab9a Mon Sep 17 00:00:00 2001 From: Haytham Abuelfutuh Date: Thu, 20 Jun 2024 06:59:44 -0700 Subject: [PATCH 3/3] wip Signed-off-by: Haytham Abuelfutuh --- s3fs/tests/test_errors.py | 24 ++++++++++++++++++++++++ s3fs/tests/test_s3fs.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 s3fs/tests/test_errors.py diff --git a/s3fs/tests/test_errors.py b/s3fs/tests/test_errors.py new file mode 100644 index 00000000..198e2b7b --- /dev/null +++ b/s3fs/tests/test_errors.py @@ -0,0 +1,24 @@ +import pytest +from s3fs.tests.test_s3fs import s3_base, s3, test_bucket_name +from s3fs import S3Map, S3FileSystem + +root = test_bucket_name + "/mapping" + + +def test_simple(s3): + d = s3.get_mapper(root) + assert not d + + assert list(d) == list(d.keys()) == [] + assert list(d.values()) == [] + assert list(d.items()) == [] + s3.get_mapper(root) + + try: + # Make an operation that raises IOError or OSError + f = d["nonexistent"] + print(f) + except OSError as e: + # verify error details + ... + diff --git a/s3fs/tests/test_s3fs.py b/s3fs/tests/test_s3fs.py index d3d90899..fc1221f6 100644 --- a/s3fs/tests/test_s3fs.py +++ b/s3fs/tests/test_s3fs.py @@ -98,7 +98,7 @@ def get_boto3_client(): # NB: we use the sync botocore client for setup session = Session() - return session.create_client("s3", endpoint_url=endpoint_uri) + return session.create_client("s3", endpoint_url=endpoint_uri, region_name="us-east-1") @pytest.fixture()