From 843b522dbbc45e5d83d0952f9315fdc4686fd61a Mon Sep 17 00:00:00 2001 From: Thomas Rausch Date: Wed, 5 Jun 2024 16:06:19 +0200 Subject: [PATCH 1/2] fix aws-replicator compatibility with localstack >=3.4 --- .github/workflows/aws-replicator.yml | 2 +- .../aws_replicator/server/aws_request_forwarder.py | 12 ++++++------ aws-replicator/aws_replicator/server/extension.py | 8 ++++++++ aws-replicator/tests/test_proxy_requests.py | 9 +++++++-- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/aws-replicator.yml b/.github/workflows/aws-replicator.yml index f972555..49bd888 100644 --- a/.github/workflows/aws-replicator.yml +++ b/.github/workflows/aws-replicator.yml @@ -57,7 +57,7 @@ jobs: find /home/runner/.cache/localstack/volume/lib/extensions/python_venv/lib/python3.11/site-packages/aws* ls -la /home/runner/.cache/localstack/volume/lib/extensions/python_venv/lib/python3.11/site-packages/aws* - DEBUG=1 localstack start -d + DEBUG=1 GATEWAY_SERVER=hypercorn localstack start -d localstack wait - name: Run linter diff --git a/aws-replicator/aws_replicator/server/aws_request_forwarder.py b/aws-replicator/aws_replicator/server/aws_request_forwarder.py index e017e99..c2e6ba1 100644 --- a/aws-replicator/aws_replicator/server/aws_request_forwarder.py +++ b/aws-replicator/aws_replicator/server/aws_request_forwarder.py @@ -6,12 +6,7 @@ import requests from localstack.aws.api import RequestContext from localstack.aws.chain import Handler, HandlerChain -from localstack.constants import ( - APPLICATION_JSON, - LOCALHOST, - LOCALHOST_HOSTNAME, - TEST_AWS_ACCESS_KEY_ID, -) +from localstack.constants import APPLICATION_JSON, LOCALHOST, LOCALHOST_HOSTNAME from localstack.http import Response from localstack.utils.aws import arns from localstack.utils.aws.arns import sqs_queue_arn @@ -22,6 +17,11 @@ from localstack.utils.strings import to_str, truncate from requests.structures import CaseInsensitiveDict +try: + from localstack.testing.config import TEST_AWS_ACCESS_KEY_ID +except ImportError: + from localstack.constants import TEST_AWS_ACCESS_KEY_ID + from aws_replicator.shared.models import ProxyInstance, ProxyServiceConfig LOG = logging.getLogger(__name__) diff --git a/aws-replicator/aws_replicator/server/extension.py b/aws-replicator/aws_replicator/server/extension.py index 7f17c5a..7fe30e0 100644 --- a/aws-replicator/aws_replicator/server/extension.py +++ b/aws-replicator/aws_replicator/server/extension.py @@ -1,5 +1,6 @@ import logging +from localstack import config from localstack.aws.chain import CompositeHandler from localstack.extensions.api import Extension, http from localstack.services.internal import get_internal_apis @@ -10,6 +11,13 @@ class AwsReplicatorExtension(Extension): name = "aws-replicator" + def on_extension_load(self): + if config.GATEWAY_SERVER == "twisted": + LOG.warning( + "AWS resource replicator: The aws-replicator extension currently requires hypercorn as " + "gateway server. Please start localstack with GATEWAY_SERVER=hypercorn" + ) + def update_gateway_routes(self, router: http.Router[http.RouteHandler]): from aws_replicator.server.request_handler import RequestHandler diff --git a/aws-replicator/tests/test_proxy_requests.py b/aws-replicator/tests/test_proxy_requests.py index afc40c5..0b3f0f1 100644 --- a/aws-replicator/tests/test_proxy_requests.py +++ b/aws-replicator/tests/test_proxy_requests.py @@ -6,7 +6,6 @@ import pytest from botocore.exceptions import ClientError from localstack.aws.connect import connect_to -from localstack.constants import TEST_AWS_ACCOUNT_ID from localstack.utils.aws.arns import sqs_queue_arn, sqs_queue_url_for_arn from localstack.utils.net import wait_for_port_open from localstack.utils.sync import retry @@ -14,6 +13,12 @@ from aws_replicator.client.auth_proxy import start_aws_auth_proxy from aws_replicator.shared.models import ProxyConfig +try: + from localstack.testing.config import TEST_AWS_ACCOUNT_ID +except ImportError: + # backwards compatibility + from localstack.constants import TEST_AWS_ACCOUNT_ID + # binding proxy to 0.0.0.0 to enable testing in CI PROXY_BIND_HOST = "0.0.0.0" @@ -34,7 +39,7 @@ def _start(config: dict = None): proxy.shutdown() -@pytest.mark.parametrize("metadata_gzip", [True, False]) +@pytest.mark.parametrize("metadata_gzip", [False]) def test_s3_requests(start_aws_proxy, s3_create_bucket, metadata_gzip): # start proxy config = ProxyConfig(services={"s3": {"resources": ".*"}}, bind_host=PROXY_BIND_HOST) From d84595de4f8747cab7c569c34a72a7029e639783 Mon Sep 17 00:00:00 2001 From: Thomas Rausch Date: Wed, 5 Jun 2024 16:11:52 +0200 Subject: [PATCH 2/2] fix an oopsie --- aws-replicator/tests/test_proxy_requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws-replicator/tests/test_proxy_requests.py b/aws-replicator/tests/test_proxy_requests.py index 0b3f0f1..f96e9ee 100644 --- a/aws-replicator/tests/test_proxy_requests.py +++ b/aws-replicator/tests/test_proxy_requests.py @@ -39,7 +39,7 @@ def _start(config: dict = None): proxy.shutdown() -@pytest.mark.parametrize("metadata_gzip", [False]) +@pytest.mark.parametrize("metadata_gzip", [True, False]) def test_s3_requests(start_aws_proxy, s3_create_bucket, metadata_gzip): # start proxy config = ProxyConfig(services={"s3": {"resources": ".*"}}, bind_host=PROXY_BIND_HOST)