Merge pull request #161 from etianen/text-mode

Adding support for opening files in text mode

Author: etianen

CHANGELOG.rst

@@ -1,6 +1,11 @@
 django-s3-storage changelog
 ===========================
 
+0.15.0
+------
+
+- Added support for opening files in text mode on Python 3.11+ (@etianen).
+
 0.14.0
 ------
 

django_s3_storage/__init__.py

@@ -3,4 +3,4 @@
 """
 
 
-__version__ = (0, 14, 0)
+__version__ = (0, 15, 0)

django_s3_storage/storage.py

@@ -6,15 +6,15 @@
 import shutil
 from contextlib import closing
 from datetime import timezone
-from functools import wraps, partial
-from io import TextIOBase
+from functools import partial, wraps
+from io import TextIOBase, TextIOWrapper
 from tempfile import SpooledTemporaryFile
 from threading import local
 from urllib.parse import urljoin, urlsplit, urlunsplit
 
 import boto3
-from botocore.client import Config
 from boto3.s3.transfer import TransferConfig
+from botocore.client import Config
 from botocore.exceptions import ClientError
 from django.conf import settings
 from django.contrib.staticfiles.storage import ManifestFilesMixin
@@ -41,6 +41,7 @@ def _do_wrap_errors(self, name, *args, **kwargs):
             if code == "NoSuchKey":
                 err_cls = FileNotFoundError
             raise err_cls(f"S3Storage error at {name!r}: {force_str(ex)}")
+
     return _do_wrap_errors
 
 
@@ -63,6 +64,7 @@ def do_wrap_path_impl(self, name, *args, **kwargs):
         # posix paths. We fix this by converting paths to system form, passing them to the default implementation, then
         # converting them back to posix paths.
         return _to_posix_path(func(self, _to_sys_path(name), *args, **kwargs))
+
     return do_wrap_path_impl
 
 
@@ -108,18 +110,24 @@ def __init__(self, storage):
         if storage.settings.AWS_ACCESS_KEY_ID:
             connection_kwargs["aws_access_key_id"] = storage.settings.AWS_ACCESS_KEY_ID
         if storage.settings.AWS_SECRET_ACCESS_KEY:
-            connection_kwargs["aws_secret_access_key"] = storage.settings.AWS_SECRET_ACCESS_KEY
+            connection_kwargs[
+                "aws_secret_access_key"
+            ] = storage.settings.AWS_SECRET_ACCESS_KEY
         if storage.settings.AWS_SESSION_TOKEN:
             connection_kwargs["aws_session_token"] = storage.settings.AWS_SESSION_TOKEN
         if storage.settings.AWS_S3_ENDPOINT_URL:
             connection_kwargs["endpoint_url"] = storage.settings.AWS_S3_ENDPOINT_URL
         self.session = boto3.session.Session()
-        self.s3_connection = self.session.client("s3", config=Config(
-            s3={"addressing_style": storage.settings.AWS_S3_ADDRESSING_STYLE},
-            signature_version=storage.settings.AWS_S3_SIGNATURE_VERSION,
-            max_pool_connections=storage.settings.AWS_S3_MAX_POOL_CONNECTIONS,
-            connect_timeout=storage.settings.AWS_S3_CONNECT_TIMEOUT
-        ), **connection_kwargs)
+        self.s3_connection = self.session.client(
+            "s3",
+            config=Config(
+                s3={"addressing_style": storage.settings.AWS_S3_ADDRESSING_STYLE},
+                signature_version=storage.settings.AWS_S3_SIGNATURE_VERSION,
+                max_pool_connections=storage.settings.AWS_S3_MAX_POOL_CONNECTIONS,
+                connect_timeout=storage.settings.AWS_S3_CONNECT_TIMEOUT,
+            ),
+            **connection_kwargs,
+        )
 
 
 @deconstructible
@@ -155,7 +163,7 @@ class S3Storage(Storage):
         "AWS_S3_FILE_OVERWRITE": False,
         "AWS_S3_USE_THREADS": True,
         "AWS_S3_MAX_POOL_CONNECTIONS": 10,
-        "AWS_S3_CONNECT_TIMEOUT": 60  # 60 seconds
+        "AWS_S3_CONNECT_TIMEOUT": 60,  # 60 seconds
     }
 
     s3_settings_suffix = ""
@@ -178,16 +186,24 @@ def _setup(self):
                 setting_key,
                 self._kwargs.get(
                     setting_key.lower(),
-                    getattr(settings, setting_key + self.s3_settings_suffix, setting_default_value),
+                    getattr(
+                        settings,
+                        setting_key + self.s3_settings_suffix,
+                        setting_default_value,
+                    ),
                 ),
             )
         # Validate settings.
         if not self.settings.AWS_S3_BUCKET_NAME:
-            raise ImproperlyConfigured(f"Setting AWS_S3_BUCKET_NAME{self.s3_settings_suffix} is required.")
+            raise ImproperlyConfigured(
+                f"Setting AWS_S3_BUCKET_NAME{self.s3_settings_suffix} is required."
+            )
         # Create a thread-local connection manager.
         self._connections = _Local(self)
         # Set transfer config for S3 operations
-        self._transfer_config = TransferConfig(use_threads=self.settings.AWS_S3_USE_THREADS)
+        self._transfer_config = TransferConfig(
+            use_threads=self.settings.AWS_S3_USE_THREADS
+        )
 
     @property
     def s3_connection(self):
@@ -201,8 +217,8 @@ def __init__(self, **kwargs):
         # Check for unknown kwargs.
         for kwarg_key in kwargs.keys():
             if (
-                kwarg_key.upper() not in self.default_auth_settings and
-                kwarg_key.upper() not in self.default_s3_settings
+                kwarg_key.upper() not in self.default_auth_settings
+                and kwarg_key.upper() not in self.default_s3_settings
             ):
                 raise ImproperlyConfigured(f"Unknown S3Storage parameter: {kwarg_key}")
         # Set up the storage.
@@ -211,7 +227,9 @@ def __init__(self, **kwargs):
         # Re-initialize the storage if an AWS setting changes.
         setting_changed.connect(self._setting_changed_received)
         # Register system checks.
-        checks.register(partial(self.__class__._system_checks, self), checks.Tags.security)
+        checks.register(
+            partial(self.__class__._system_checks, self), checks.Tags.security
+        )
         # All done!
         super().__init__()
 
@@ -237,7 +255,9 @@ def __reduce__(self):
     def _get_key_name(self, name):
         if name.startswith("/"):
             name = name[1:]
-        return posixpath.normpath(posixpath.join(self.settings.AWS_S3_KEY_PREFIX, _to_posix_path(name)))
+        return posixpath.normpath(
+            posixpath.join(self.settings.AWS_S3_KEY_PREFIX, _to_posix_path(name))
+        )
 
     def _object_params(self, name):
         params = {
@@ -256,22 +276,29 @@ def _object_put_params(self, name):
             ),
             "Metadata": {
                 key: _callable_setting(value, name)
-                for key, value
-                in self.settings.AWS_S3_METADATA.items()
+                for key, value in self.settings.AWS_S3_METADATA.items()
             },
-            "StorageClass": "REDUCED_REDUNDANCY" if self.settings.AWS_S3_REDUCED_REDUNDANCY else "STANDARD",
+            "StorageClass": "REDUCED_REDUNDANCY"
+            if self.settings.AWS_S3_REDUCED_REDUNDANCY
+            else "STANDARD",
         }
         params.update(self._object_params(name))
         # Set content disposition.
-        content_disposition = _callable_setting(self.settings.AWS_S3_CONTENT_DISPOSITION, name)
+        content_disposition = _callable_setting(
+            self.settings.AWS_S3_CONTENT_DISPOSITION, name
+        )
         if content_disposition:
             params["ContentDisposition"] = content_disposition
         # Set content langauge.
-        content_langauge = _callable_setting(self.settings.AWS_S3_CONTENT_LANGUAGE, name)
+        content_langauge = _callable_setting(
+            self.settings.AWS_S3_CONTENT_LANGUAGE, name
+        )
         if content_langauge:
             params["ContentLanguage"] = content_langauge
         # Set server-side encryption.
-        if self.settings.AWS_S3_ENCRYPT_KEY:  # If this if False / None / empty then no encryption
+        if (
+            self.settings.AWS_S3_ENCRYPT_KEY
+        ):  # If this if False / None / empty then no encryption
             if isinstance(self.settings.AWS_S3_ENCRYPT_KEY, str):
                 params["ServerSideEncryption"] = self.settings.AWS_S3_ENCRYPT_KEY
                 if self.settings.AWS_S3_KMS_ENCRYPTION_KEY_ID:
@@ -283,11 +310,11 @@ def _object_put_params(self, name):
 
     def new_temporary_file(self):
         """Returns a new file to use when opening from or saving to S3"""
-        return SpooledTemporaryFile(max_size=1024*1024*10)  # 10 MB.
+        return SpooledTemporaryFile(max_size=1024 * 1024 * 10)  # 10 MB.
 
     @_wrap_errors
     def _open(self, name, mode="rb"):
-        if mode != "rb":
+        if mode not in ("rb", "rt", "r"):
             raise ValueError("S3 files can only be opened in read-only mode")
         # Load the key into a temporary file. It would be nice to stream the
         # content, but S3 doesn't support seeking, which is sometimes needed.
@@ -298,6 +325,9 @@ def _open(self, name, mode="rb"):
         # Un-gzip if required.
         if obj.get("ContentEncoding") == "gzip":
             content = gzip.GzipFile(name, "rb", fileobj=content)
+        # Decode text if required.
+        if "b" not in mode:
+            content = TextIOWrapper(content)
         # All done!
         return S3File(content, name, self)
 
@@ -325,7 +355,12 @@ def _save(self, name, content):
             # Check if the content type is compressible.
             content_type_family, content_type_subtype = content_type.lower().split("/")
             content_type_subtype = content_type_subtype.split("+")[-1]
-            if content_type_family == "text" or content_type_subtype in ("xml", "json", "html", "javascript"):
+            if content_type_family == "text" or content_type_subtype in (
+                "xml",
+                "json",
+                "html",
+                "javascript",
+            ):
                 # Compress the content.
                 temp_file = self.new_temporary_file()
                 temp_files.append(temp_file)
@@ -337,7 +372,9 @@ def _save(self, name, content):
                     temp_file.seek(0)
                     content = temp_file
                     put_params["ContentEncoding"] = "gzip"
-                    put_params["Metadata"][_UNCOMPRESSED_SIZE_META_KEY] = f"{orig_size:d}"
+                    put_params["Metadata"][
+                        _UNCOMPRESSED_SIZE_META_KEY
+                    ] = f"{orig_size:d}"
                 else:
                     content.seek(0)
         # Save the file.
@@ -346,8 +383,13 @@ def _save(self, name, content):
         original_close = content.close
         content.close = lambda: None
         try:
-            self.s3_connection.upload_fileobj(content, put_params.pop('Bucket'), put_params.pop('Key'),
-                                              ExtraArgs=put_params, Config=self._transfer_config)
+            self.s3_connection.upload_fileobj(
+                content,
+                put_params.pop("Bucket"),
+                put_params.pop("Key"),
+                ExtraArgs=put_params,
+                Config=self._transfer_config,
+            )
         finally:
             # Restore the original close method.
             content.close = original_close
@@ -385,8 +427,8 @@ def delete(self, name):
     @_wrap_errors
     def copy(self, src_name, dst_name):
         self.s3_connection.copy_object(
-            CopySource=self._object_params(src_name),
-            **self._object_params(dst_name))
+            CopySource=self._object_params(src_name), **self._object_params(dst_name)
+        )
 
     @_wrap_errors
     def rename(self, src_name, dst_name):
@@ -402,7 +444,8 @@ def exists(self, name):
                 results = self.s3_connection.list_objects_v2(
                     Bucket=self.settings.AWS_S3_BUCKET_NAME,
                     MaxKeys=1,
-                    Prefix=self._get_key_name(name) + "/",  # Add the slash again, since _get_key_name removes it.
+                    Prefix=self._get_key_name(name)
+                    + "/",  # Add the slash again, since _get_key_name removes it.
                 )
             except ClientError:
                 return False
@@ -449,7 +492,9 @@ def url(self, name, extra_params=None, client_method="get_object"):
         # Use a public URL, if specified.
         if self.settings.AWS_S3_PUBLIC_URL:
             if extra_params or client_method != "get_object":
-                raise ValueError("Use of extra_params or client_method is not allowed with AWS_S3_PUBLIC_URL")
+                raise ValueError(
+                    "Use of extra_params or client_method is not allowed with AWS_S3_PUBLIC_URL"
+                )
             return urljoin(self.settings.AWS_S3_PUBLIC_URL, filepath_to_uri(name))
         # Otherwise, generate the URL.
         params = extra_params.copy() if extra_params else {}
@@ -462,8 +507,16 @@ def url(self, name, extra_params=None, client_method="get_object"):
         # Strip off the query params if we're not interested in bucket auth.
         if not self.settings.AWS_S3_BUCKET_AUTH:
             if extra_params or client_method != "get_object":
-                raise ValueError("Use of extra_params or client_method is not allowed with AWS_S3_BUCKET_AUTH")
-            url = urlunsplit(urlsplit(url)[:3] + ("", "",))
+                raise ValueError(
+                    "Use of extra_params or client_method is not allowed with AWS_S3_BUCKET_AUTH"
+                )
+            url = urlunsplit(
+                urlsplit(url)[:3]
+                + (
+                    "",
+                    "",
+                )
+            )
         # All done!
         return url
 
@@ -501,8 +554,9 @@ def sync_meta_iter(self):
                     put_params["ContentEncoding"] = content_encoding
                     if content_encoding == "gzip":
                         try:
-                            put_params["Metadata"][_UNCOMPRESSED_SIZE_META_KEY] = \
-                                obj["Metadata"][_UNCOMPRESSED_SIZE_META_KEY]
+                            put_params["Metadata"][_UNCOMPRESSED_SIZE_META_KEY] = obj[
+                                "Metadata"
+                            ][_UNCOMPRESSED_SIZE_META_KEY]
                         except KeyError:
                             pass
                 # Update the metadata.
@@ -513,7 +567,7 @@ def sync_meta_iter(self):
                         "Key": self._get_key_name(name),
                     },
                     MetadataDirective="REPLACE",
-                    **put_params
+                    **put_params,
                 )
                 yield name
 
@@ -529,23 +583,28 @@ class StaticS3Storage(S3Storage):
     """
 
     default_s3_settings = S3Storage.default_s3_settings.copy()
-    default_s3_settings.update({
-        "AWS_S3_BUCKET_AUTH": False,
-    })
+    default_s3_settings.update(
+        {
+            "AWS_S3_BUCKET_AUTH": False,
+        }
+    )
 
     s3_settings_suffix = "_STATIC"
 
 
 class ManifestStaticS3Storage(ManifestFilesMixin, StaticS3Storage):
-
     default_s3_settings = StaticS3Storage.default_s3_settings.copy()
-    default_s3_settings.update({
-        "AWS_S3_MAX_AGE_SECONDS_CACHED": 60 * 60 * 24 * 365,  # 1 year.
-    })
+    default_s3_settings.update(
+        {
+            "AWS_S3_MAX_AGE_SECONDS_CACHED": 60 * 60 * 24 * 365,  # 1 year.
+        }
+    )
 
     def post_process(self, *args, **kwargs):
         initial_aws_s3_max_age_seconds = self.settings.AWS_S3_MAX_AGE_SECONDS
-        self.settings.AWS_S3_MAX_AGE_SECONDS = self.settings.AWS_S3_MAX_AGE_SECONDS_CACHED
+        self.settings.AWS_S3_MAX_AGE_SECONDS = (
+            self.settings.AWS_S3_MAX_AGE_SECONDS_CACHED
+        )
         try:
             yield from super().post_process(*args, **kwargs)
         finally: