Workaround for botocore 1.28 (round 2) (#1087)

In #1083 we've started passing an `endpoint_url` parameter to _convert_to_request_dict due to changes made in botocore 1.28.

When a model does not specify a `host`, the `endpoint_url` would be `None`. To determine the actual `endpoint_url` in botocore ≥1.28, we must call another private method, `_resolve_endpoint_ruleset`.

Author: ikonst

docs/release_notes.rst

@@ -1,6 +1,13 @@
 Release Notes
 =============
 
+v5.2.3
+----------
+* Update for botocore 1.28 private API change (#1087) which caused the following exception::
+
+    TypeError: Cannot mix str and non-str arguments
+
+
 v5.2.2
 ----------
 * Update for botocore 1.28 private API change (#1083) which caused the following exception::

pynamodb/__init__.py

@@ -7,4 +7,4 @@
 """
 __author__ = 'Jharrod LaFon'
 __license__ = 'MIT'
-__version__ = '5.2.2'
+__version__ = '5.2.3'

pynamodb/connection/_botocore_private.py

@@ -1,7 +1,7 @@
 """
 Type-annotates the private botocore APIs that we're currently relying on.
 """
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 import botocore.client
 import botocore.credentials
@@ -24,5 +24,23 @@ class BotocoreBaseClientPrivate(botocore.client.BaseClient):
     _request_signer: BotocoreRequestSignerPrivate
     _service_model: botocore.model.ServiceModel
 
-    def _convert_to_request_dict(self, api_params: Dict[str, Any], operation_model: botocore.model.OperationModel, *args: Any, **kwargs: Any) -> Dict[str, Any]:
+    def _resolve_endpoint_ruleset(
+        self,
+        operation_model: botocore.model.OperationModel,
+        params: Dict[str, Any],
+        request_context: Dict[str, Any],
+        ignore_signing_region: bool = ...,
+    ):
+        ...
+
+    def _convert_to_request_dict(
+        self,
+        api_params: Dict[str, Any],
+        operation_model: botocore.model.OperationModel,
+        *,
+        endpoint_url: str = ...,  # added in botocore 1.28
+        context: Optional[Dict[str, Any]] = ...,
+        headers: Optional[Dict[str, Any]] = ...,
+        set_user_agent_header: bool = ...,
+    ) -> Dict[str, Any]:
         ...

pynamodb/connection/base.py

@@ -254,7 +254,7 @@ def __init__(self,
         self.host = host
         self._local = local()
         self._client: Optional[BotocoreBaseClientPrivate] = None
-        self._convert_to_request_dict_kwargs: Dict[str, Any] = {}
+        self._convert_to_request_dict__endpoint_url = False
         if region:
             self.region = region
         else:
@@ -358,11 +358,28 @@ def _make_api_call(self, operation_name: str, operation_kwargs: Dict, settings:
         2. It provides a place to monkey patch HTTP requests for unit testing
         """
         operation_model = self.client._service_model.operation_model(operation_name)
-        request_dict = self.client._convert_to_request_dict(
-            operation_kwargs,
-            operation_model,
-            **self._convert_to_request_dict_kwargs,
-        )
+        if self._convert_to_request_dict__endpoint_url:
+            request_context = {
+                'client_region': self.region,
+                'client_config': self.client.meta.config,
+                'has_streaming_input': operation_model.has_streaming_input,
+                'auth_type': operation_model.auth_type,
+            }
+            endpoint_url, additional_headers = self.client._resolve_endpoint_ruleset(
+                operation_model, operation_kwargs, request_context
+            )
+            request_dict = self.client._convert_to_request_dict(
+                api_params=operation_kwargs,
+                operation_model=operation_model,
+                endpoint_url=endpoint_url,
+                context=request_context,
+                headers=additional_headers,
+            )
+        else:
+            request_dict = self.client._convert_to_request_dict(
+                operation_kwargs,
+                operation_model,
+            )
 
         for i in range(0, self._max_retry_attempts_exception + 1):
             attempt_number = i + 1
@@ -536,11 +553,10 @@ def client(self) -> BotocoreBaseClientPrivate:
                 parameter_validation=False,  # Disable unnecessary validation for performance
                 connect_timeout=self._connect_timeout_seconds,
                 read_timeout=self._read_timeout_seconds,
-                max_pool_connections=self._max_pool_connections)
+                max_pool_connections=self._max_pool_connections,
+            )
             self._client = cast(BotocoreBaseClientPrivate, self.session.create_client(SERVICE_NAME, self.region, endpoint_url=self.host, config=config))
-            self._convert_to_request_dict_kwargs = {}
-            if 'endpoint_url' in inspect.signature(self._client._convert_to_request_dict).parameters:
-                self._convert_to_request_dict_kwargs['endpoint_url'] = self.host
+            self._convert_to_request_dict__endpoint_url = 'endpoint_url' in inspect.signature(self._client._convert_to_request_dict).parameters
         return self._client
 
     def get_meta_table(self, table_name: str, refresh: bool = False):

tests/test_base_connection.py

@@ -2,11 +2,14 @@
 Tests for the base connection class
 """
 import base64
+import io
 import json
 from unittest import mock, TestCase
 from unittest.mock import patch
 
 import botocore.exceptions
+import botocore.httpsession
+import urllib3
 from botocore.awsrequest import AWSPreparedRequest, AWSRequest, AWSResponse
 from botocore.client import ClientError
 from botocore.exceptions import BotoCoreError
@@ -1398,6 +1401,22 @@ def test_scan(self):
                 conn.scan,
                 table_name)
 
+    def test_make_api_call__happy_path(self):
+        response = AWSResponse(
+            url='https://www.example.com',
+            status_code=200,
+            raw=urllib3.HTTPResponse(
+                body=io.BytesIO(json.dumps({}).encode('utf-8')),
+                preload_content=False,
+            ),
+            headers={'x-amzn-RequestId': 'abcdef'},
+        )
+
+        c = Connection()
+
+        with patch.object(botocore.httpsession.URLLib3Session, 'send', return_value=response):
+            c._make_api_call('CreateTable', {'TableName': 'MyTable'})
+
     @mock.patch('pynamodb.connection.Connection.client')
     def test_make_api_call_throws_verbose_error_after_backoff(self, client_mock):
         response = AWSResponse(