|
43 | 43 | UNIFIED_AWS_SESSION_PROPERTIES, |
44 | 44 | ) |
45 | 45 |
|
| 46 | +S3TABLES_WAREHOUSE_LOCATION = "s3tables-warehouse-location" |
| 47 | + |
| 48 | + |
| 49 | +def _patch_moto_for_s3tables(monkeypatch: pytest.MonkeyPatch) -> None: |
| 50 | + """Patch moto to simulate S3 Tables federated databases. |
| 51 | +
|
| 52 | + Moto does not support FederatedDatabase on GetDatabase responses or |
| 53 | + auto-populating StorageDescriptor.Location for S3 Tables. These patches |
| 54 | + simulate the S3 Tables service behavior so that the GlueCatalog S3 Tables |
| 55 | + code path can be tested end-to-end with moto. |
| 56 | + """ |
| 57 | + from moto.glue.models import FakeDatabase, FakeTable |
| 58 | + |
| 59 | + # Patch 1: Make GetDatabase return FederatedDatabase from the stored input. |
| 60 | + _original_db_as_dict = FakeDatabase.as_dict |
| 61 | + |
| 62 | + def _db_as_dict_with_federated(self): # type: ignore |
| 63 | + result = _original_db_as_dict(self) |
| 64 | + if federated := self.input.get("FederatedDatabase"): |
| 65 | + result["FederatedDatabase"] = federated |
| 66 | + return result |
| 67 | + |
| 68 | + monkeypatch.setattr(FakeDatabase, "as_dict", _db_as_dict_with_federated) |
| 69 | + |
| 70 | + # Patch 2: When a table is created with format=ICEBERG (the S3 Tables convention), |
| 71 | + # inject a StorageDescriptor.Location to simulate S3 Tables vending a table |
| 72 | + # warehouse location. |
| 73 | + _original_table_init = FakeTable.__init__ |
| 74 | + |
| 75 | + def _table_init_with_location(self, database_name, table_name, table_input, catalog_id): # type: ignore |
| 76 | + if table_input.get("Parameters", {}).get("format") == "ICEBERG" and "StorageDescriptor" not in table_input: |
| 77 | + table_input = { |
| 78 | + **table_input, |
| 79 | + "StorageDescriptor": { |
| 80 | + "Columns": [], |
| 81 | + "Location": f"s3://{S3TABLES_WAREHOUSE_LOCATION}/{database_name}/{table_name}/", |
| 82 | + "InputFormat": "", |
| 83 | + "OutputFormat": "", |
| 84 | + "SerdeInfo": {}, |
| 85 | + }, |
| 86 | + } |
| 87 | + _original_table_init(self, database_name, table_name, table_input, catalog_id) |
| 88 | + |
| 89 | + monkeypatch.setattr(FakeTable, "__init__", _table_init_with_location) |
| 90 | + |
| 91 | + # Create a bucket backing the simulated table warehouse location. S3 Tables manages |
| 92 | + # this storage internally, but in tests moto needs a real bucket for metadata file |
| 93 | + # writes to succeed. |
| 94 | + s3 = boto3.client("s3", region_name="us-east-1") |
| 95 | + s3.create_bucket(Bucket=S3TABLES_WAREHOUSE_LOCATION) |
| 96 | + |
46 | 97 |
|
47 | 98 | @mock_aws |
48 | 99 | def test_create_table_with_database_location( |
@@ -953,3 +1004,78 @@ def test_glue_client_override() -> None: |
953 | 1004 | test_client = boto3.client("glue", region_name="us-west-2") |
954 | 1005 | test_catalog = GlueCatalog(catalog_name, test_client) |
955 | 1006 | assert test_catalog.glue is test_client |
| 1007 | + |
| 1008 | + |
| 1009 | +def _create_s3tables_database(catalog: GlueCatalog, database_name: str) -> None: |
| 1010 | + """Create a Glue database with S3 Tables federation metadata.""" |
| 1011 | + catalog.glue.create_database( |
| 1012 | + DatabaseInput={ |
| 1013 | + "Name": database_name, |
| 1014 | + "FederatedDatabase": { |
| 1015 | + "Identifier": "arn:aws:s3tables:us-east-1:123456789012:bucket/my-bucket", |
| 1016 | + "ConnectionType": "aws:s3tables", |
| 1017 | + }, |
| 1018 | + } |
| 1019 | + ) |
| 1020 | + |
| 1021 | + |
| 1022 | +@mock_aws |
| 1023 | +def test_create_table_s3tables( |
| 1024 | + monkeypatch: pytest.MonkeyPatch, |
| 1025 | + _bucket_initialize: None, |
| 1026 | + moto_endpoint_url: str, |
| 1027 | + table_schema_nested: Schema, |
| 1028 | + database_name: str, |
| 1029 | + table_name: str, |
| 1030 | +) -> None: |
| 1031 | + _patch_moto_for_s3tables(monkeypatch) |
| 1032 | + |
| 1033 | + identifier = (database_name, table_name) |
| 1034 | + test_catalog = GlueCatalog("s3tables", **{"s3.endpoint": moto_endpoint_url}) |
| 1035 | + _create_s3tables_database(test_catalog, database_name) |
| 1036 | + |
| 1037 | + table = test_catalog.create_table(identifier, table_schema_nested) |
| 1038 | + assert table.name() == identifier |
| 1039 | + assert table.location() == f"s3://{S3TABLES_WAREHOUSE_LOCATION}/{database_name}/{table_name}" |
| 1040 | + assert table.metadata_location.startswith(f"s3://{S3TABLES_WAREHOUSE_LOCATION}/{database_name}/{table_name}/metadata/00000-") |
| 1041 | + assert table.metadata_location.endswith(".metadata.json") |
| 1042 | + assert test_catalog._parse_metadata_version(table.metadata_location) == 0 |
| 1043 | + |
| 1044 | + |
| 1045 | +@mock_aws |
| 1046 | +def test_create_table_s3tables_rejects_location( |
| 1047 | + monkeypatch: pytest.MonkeyPatch, |
| 1048 | + _bucket_initialize: None, |
| 1049 | + moto_endpoint_url: str, |
| 1050 | + table_schema_nested: Schema, |
| 1051 | + database_name: str, |
| 1052 | + table_name: str, |
| 1053 | +) -> None: |
| 1054 | + _patch_moto_for_s3tables(monkeypatch) |
| 1055 | + |
| 1056 | + identifier = (database_name, table_name) |
| 1057 | + test_catalog = GlueCatalog("s3tables", **{"s3.endpoint": moto_endpoint_url}) |
| 1058 | + _create_s3tables_database(test_catalog, database_name) |
| 1059 | + |
| 1060 | + with pytest.raises(ValueError, match="Cannot specify a location for S3 Tables table"): |
| 1061 | + test_catalog.create_table(identifier, table_schema_nested, location="s3://some-bucket/some-path") |
| 1062 | + |
| 1063 | + |
| 1064 | +@mock_aws |
| 1065 | +def test_create_table_s3tables_duplicate( |
| 1066 | + monkeypatch: pytest.MonkeyPatch, |
| 1067 | + _bucket_initialize: None, |
| 1068 | + moto_endpoint_url: str, |
| 1069 | + table_schema_nested: Schema, |
| 1070 | + database_name: str, |
| 1071 | + table_name: str, |
| 1072 | +) -> None: |
| 1073 | + _patch_moto_for_s3tables(monkeypatch) |
| 1074 | + |
| 1075 | + identifier = (database_name, table_name) |
| 1076 | + test_catalog = GlueCatalog("s3tables", **{"s3.endpoint": moto_endpoint_url}) |
| 1077 | + _create_s3tables_database(test_catalog, database_name) |
| 1078 | + |
| 1079 | + test_catalog.create_table(identifier, table_schema_nested) |
| 1080 | + with pytest.raises(TableAlreadyExistsError): |
| 1081 | + test_catalog.create_table(identifier, table_schema_nested) |
0 commit comments