Merge pull request #9370 from AndrewAsseily/nyandrew/ruff-format-base-repo-customizations-nested-s3

Run ruff format

Author: AndrewAsseily

awscli/customizations/s3/comparator.py

@@ -21,10 +21,13 @@ class Comparator(object):
     """
     This class performs all of the comparisons behind the sync operation
     """
-    def __init__(self, file_at_src_and_dest_sync_strategy,
-                 file_not_at_dest_sync_strategy,
-                 file_not_at_src_sync_strategy):
 
+    def __init__(
+        self,
+        file_at_src_and_dest_sync_strategy,
+        file_not_at_dest_sync_strategy,
+        file_not_at_src_sync_strategy,
+    ):
         self._sync_strategy = file_at_src_and_dest_sync_strategy
         self._not_at_dest_sync_strategy = file_not_at_dest_sync_strategy
         self._not_at_src_sync_strategy = file_not_at_src_sync_strategy
@@ -102,26 +105,42 @@ def call(self, src_files, dest_files):
                 elif compare_keys == 'less_than':
                     src_take = True
                     dest_take = False
-                    should_sync = self._not_at_dest_sync_strategy.determine_should_sync(src_file, None)
+                    should_sync = (
+                        self._not_at_dest_sync_strategy.determine_should_sync(
+                            src_file, None
+                        )
+                    )
                     if should_sync:
                         yield src_file
 
                 elif compare_keys == 'greater_than':
                     src_take = False
                     dest_take = True
-                    should_sync = self._not_at_src_sync_strategy.determine_should_sync(None, dest_file)
+                    should_sync = (
+                        self._not_at_src_sync_strategy.determine_should_sync(
+                            None, dest_file
+                        )
+                    )
                     if should_sync:
                         yield dest_file
 
             elif (not src_done) and dest_done:
                 src_take = True
-                should_sync = self._not_at_dest_sync_strategy.determine_should_sync(src_file, None)
+                should_sync = (
+                    self._not_at_dest_sync_strategy.determine_should_sync(
+                        src_file, None
+                    )
+                )
                 if should_sync:
                     yield src_file
 
             elif src_done and (not dest_done):
                 dest_take = True
-                should_sync = self._not_at_src_sync_strategy.determine_should_sync(None, dest_file)
+                should_sync = (
+                    self._not_at_src_sync_strategy.determine_should_sync(
+                        None, dest_file
+                    )
+                )
                 if should_sync:
                     yield dest_file
             else:
@@ -135,10 +154,10 @@ def compare_comp_key(self, src_file, dest_file):
 
         src_comp_key = src_file.compare_key
         dest_comp_key = dest_file.compare_key
-        if (src_comp_key == dest_comp_key):
+        if src_comp_key == dest_comp_key:
             return 'equal'
 
-        elif (src_comp_key < dest_comp_key):
+        elif src_comp_key < dest_comp_key:
             return 'less_than'
 
         else:

awscli/customizations/s3/factory.py

@@ -17,15 +17,18 @@
 from botocore.httpsession import DEFAULT_CA_BUNDLE
 from s3transfer.manager import TransferManager
 from s3transfer.crt import (
-    acquire_crt_s3_process_lock, create_s3_crt_client,
-    BotocoreCRTRequestSerializer, CRTTransferManager,
-    BotocoreCRTCredentialsWrapper
+    acquire_crt_s3_process_lock,
+    create_s3_crt_client,
+    BotocoreCRTRequestSerializer,
+    CRTTransferManager,
+    BotocoreCRTCredentialsWrapper,
 )
 
 from awscli.compat import urlparse
 from awscli.customizations.s3 import constants
-from awscli.customizations.s3.transferconfig import \
-    create_transfer_config_from_runtime_config
+from awscli.customizations.s3.transferconfig import (
+    create_transfer_config_from_runtime_config,
+)
 
 
 LOGGER = logging.getLogger(__name__)
@@ -36,9 +39,7 @@ def __init__(self, session):
         self._session = session
 
     def create_client(self, params, is_source_client=False):
-        create_client_kwargs = {
-            'verify': params['verify_ssl']
-        }
+        create_client_kwargs = {'verify': params['verify_ssl']}
         if params.get('sse') == 'aws:kms':
             create_client_kwargs['config'] = Config(signature_version='s3v4')
         region = params['region']
@@ -61,22 +62,24 @@ def __init__(self, session):
         self._session = session
         self._botocore_client_factory = ClientFactory(self._session)
 
-    def create_transfer_manager(self, params, runtime_config,
-                                botocore_client=None):
+    def create_transfer_manager(
+        self, params, runtime_config, botocore_client=None
+    ):
         client_type = self._compute_transfer_client_type(
-            params, runtime_config)
+            params, runtime_config
+        )
         if client_type == constants.CRT_TRANSFER_CLIENT:
             return self._create_crt_transfer_manager(params, runtime_config)
         else:
             return self._create_classic_transfer_manager(
-                params, runtime_config, botocore_client)
+                params, runtime_config, botocore_client
+            )
 
     def _compute_transfer_client_type(self, params, runtime_config):
         if params.get('paths_type') == 's3s3':
             return constants.CLASSIC_TRANSFER_CLIENT
         preferred_transfer_client = runtime_config.get(
-            'preferred_transfer_client',
-            constants.AUTO_RESOLVE_TRANSFER_CLIENT
+            'preferred_transfer_client', constants.AUTO_RESOLVE_TRANSFER_CLIENT
         )
         if preferred_transfer_client == constants.AUTO_RESOLVE_TRANSFER_CLIENT:
             return self._resolve_transfer_client_type_for_system()
@@ -92,7 +95,7 @@ def _resolve_transfer_client_type_for_system(self):
             is_running = self._is_crt_client_running_in_other_aws_cli_process()
             LOGGER.debug(
                 'S3 CRT client running in different AWS CLI process: %s',
-                is_running
+                is_running,
             )
             if not is_running:
                 transfer_client_type = constants.CRT_TRANSFER_CLIENT
@@ -114,7 +117,7 @@ def _create_crt_transfer_manager(self, params, runtime_config):
         self._acquire_crt_s3_process_lock()
         return CRTTransferManager(
             self._create_crt_client(params, runtime_config),
-            self._create_crt_request_serializer(params)
+            self._create_crt_request_serializer(params),
         )
 
     def _create_crt_client(self, params, runtime_config):
@@ -133,8 +136,9 @@ def _create_crt_client(self, params, runtime_config):
             create_crt_client_kwargs['part_size'] = multipart_chunksize
         if params.get('sign_request', True):
             crt_credentials_provider = self._get_crt_credentials_provider()
-            create_crt_client_kwargs[
-                'crt_credentials_provider'] = crt_credentials_provider
+            create_crt_client_kwargs['crt_credentials_provider'] = (
+                crt_credentials_provider
+            )
 
         return create_s3_crt_client(**create_crt_client_kwargs)
 
@@ -144,23 +148,27 @@ def _create_crt_request_serializer(self, params):
             {
                 'region_name': self._resolve_region(params),
                 'endpoint_url': params.get('endpoint_url'),
-            }
+            },
         )
 
-    def _create_classic_transfer_manager(self, params, runtime_config,
-                                         client=None):
+    def _create_classic_transfer_manager(
+        self, params, runtime_config, client=None
+    ):
         if client is None:
             client = self._botocore_client_factory.create_client(params)
         transfer_config = create_transfer_config_from_runtime_config(
-            runtime_config)
-        transfer_config.max_in_memory_upload_chunks = \
+            runtime_config
+        )
+        transfer_config.max_in_memory_upload_chunks = (
             self._MAX_IN_MEMORY_CHUNKS
-        transfer_config.max_in_memory_download_chunks = \
+        )
+        transfer_config.max_in_memory_download_chunks = (
             self._MAX_IN_MEMORY_CHUNKS
+        )
         LOGGER.debug(
             "Using a multipart threshold of %s and a part size of %s",
             transfer_config.multipart_threshold,
-            transfer_config.multipart_chunksize
+            transfer_config.multipart_chunksize,
         )
         return TransferManager(client, transfer_config)
 

awscli/customizations/s3/fileformat.py

@@ -53,9 +53,12 @@ def format(self, src, dest, parameters):
         #     will take on the name the user specified in the
         #     command line.
         dest_path, use_src_name = format_table[dest_type](dest_path, dir_op)
-        files = {'src': {'path': src_path, 'type': src_type},
-                 'dest': {'path': dest_path, 'type': dest_type},
-                 'dir_op': dir_op, 'use_src_name': use_src_name}
+        files = {
+            'src': {'path': src_path, 'type': src_type},
+            'dest': {'path': dest_path, 'type': dest_type},
+            'dir_op': dir_op,
+            'use_src_name': use_src_name,
+        }
         return files
 
     def local_format(self, path, dir_op):

awscli/customizations/s3/filegenerator.py

@@ -19,8 +19,12 @@
 from botocore.exceptions import ClientError
 
 from awscli.customizations.s3.utils import find_bucket_key, get_file_stat
-from awscli.customizations.s3.utils import BucketLister, create_warning, \
-    find_dest_path_comp_key, EPOCH_TIME
+from awscli.customizations.s3.utils import (
+    BucketLister,
+    create_warning,
+    find_dest_path_comp_key,
+    EPOCH_TIME,
+)
 from awscli.compat import queue
 
 _open = open
@@ -70,6 +74,7 @@ def is_readable(path):
 
 # This class is provided primarily to provide a detailed error message.
 
+
 class FileDecodingError(Exception):
     """Raised when there was an issue decoding the file."""
 
@@ -84,17 +89,25 @@ def __init__(self, directory, filename):
         self.file_name = filename
         self.error_message = (
             'There was an error trying to decode the the file %s in '
-            'directory "%s". \n%s' % (repr(self.file_name),
-                                      self.directory,
-                                      self.ADVICE)
+            'directory "%s". \n%s'
+            % (repr(self.file_name), self.directory, self.ADVICE)
         )
         super(FileDecodingError, self).__init__(self.error_message)
 
 
 class FileStat(object):
-    def __init__(self, src, dest=None, compare_key=None, size=None,
-                 last_update=None, src_type=None, dest_type=None,
-                 operation_name=None, response_data=None):
+    def __init__(
+        self,
+        src,
+        dest=None,
+        compare_key=None,
+        size=None,
+        last_update=None,
+        src_type=None,
+        dest_type=None,
+        operation_name=None,
+        response_data=None,
+    ):
         self.src = src
         self.dest = dest
         self.compare_key = compare_key
@@ -114,8 +127,16 @@ class FileGenerator(object):
     under the same common prefix.  The generator yields corresponding
     ``FileInfo`` objects to send to a ``Comparator`` or ``S3Handler``.
     """
-    def __init__(self, client, operation_name, follow_symlinks=True,
-                 page_size=None, result_queue=None, request_parameters=None):
+
+    def __init__(
+        self,
+        client,
+        operation_name,
+        follow_symlinks=True,
+        page_size=None,
+        result_queue=None,
+        request_parameters=None,
+    ):
         self._client = client
         self.operation_name = operation_name
         self.follow_symlinks = follow_symlinks
@@ -141,9 +162,12 @@ def call(self, files):
         for src_path, extra_information in file_iterator:
             dest_path, compare_key = find_dest_path_comp_key(files, src_path)
             file_stat_kwargs = {
-                'src': src_path, 'dest': dest_path, 'compare_key': compare_key,
-                'src_type': src_type, 'dest_type': dest_type,
-                'operation_name': self.operation_name
+                'src': src_path,
+                'dest': dest_path,
+                'compare_key': compare_key,
+                'src_type': src_type,
+                'dest_type': dest_type,
+                'operation_name': self.operation_name,
             }
             self._inject_extra_information(file_stat_kwargs, extra_information)
             yield FileStat(**file_stat_kwargs)
@@ -188,7 +212,8 @@ def list_files(self, path, dir_op):
                 names = []
                 for name in listdir_names:
                     if not self.should_ignore_file_with_decoding_warnings(
-                            path, name):
+                        path, name
+                    ):
                         file_path = join(path, name)
                         if isdir(file_path):
                             name = name + os.path.sep
@@ -225,8 +250,9 @@ def _validate_update_time(self, update_time, path):
             warning = create_warning(
                 path=path,
                 error_message="File has an invalid timestamp. Passing epoch "
-                              "time as timestamp.",
-                skip_file=False)
+                "time as timestamp.",
+                skip_file=False,
+            )
             self.result_queue.put(warning)
             return EPOCH_TIME
         return update_time
@@ -251,8 +277,9 @@ def should_ignore_file_with_decoding_warnings(self, dirname, filename):
         """
         if not isinstance(filename, str):
             decoding_error = FileDecodingError(dirname, filename)
-            warning = create_warning(repr(filename),
-                                     decoding_error.error_message)
+            warning = create_warning(
+                repr(filename), decoding_error.error_message
+            )
             self.result_queue.put(warning)
             return True
         path = os.path.join(dirname, filename)
@@ -290,10 +317,14 @@ def triggers_warning(self, path):
             self.result_queue.put(warning)
             return True
         if is_special_file(path):
-            warning = create_warning(path,
-                                     ("File is character special device, "
-                                      "block special device, FIFO, or "
-                                      "socket."))
+            warning = create_warning(
+                path,
+                (
+                    "File is character special device, "
+                    "block special device, FIFO, or "
+                    "socket."
+                ),
+            )
             self.result_queue.put(warning)
             return True
         if not is_readable(path):
@@ -318,9 +349,12 @@ def list_objects(self, s3_path, dir_op):
         else:
             lister = BucketLister(self._client)
             extra_args = self.request_parameters.get('ListObjectsV2', {})
-            for key in lister.list_objects(bucket=bucket, prefix=prefix,
-                                           page_size=self.page_size,
-                                           extra_args=extra_args):
+            for key in lister.list_objects(
+                bucket=bucket,
+                prefix=prefix,
+                page_size=self.page_size,
+                extra_args=extra_args,
+            ):
                 source_path, response_data = key
                 if response_data['Size'] == 0 and source_path.endswith('/'):
                     if self.operation_name == 'delete':