diff --git a/plugins/spark/v3.5/regtests/docker-compose.yml b/plugins/spark/v3.5/regtests/docker-compose.yml index e1ea1a898..aa0c259fe 100755 --- a/plugins/spark/v3.5/regtests/docker-compose.yml +++ b/plugins/spark/v3.5/regtests/docker-compose.yml @@ -28,6 +28,9 @@ services: POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,secret quarkus.log.file.enable: "false" quarkus.otel.sdk.disabled: "true" + polaris.features."ALLOW_INSECURE_STORAGE_TYPES": "true" + polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES": "[\"FILE\",\"S3\",\"GCS\",\"AZURE\"]" + polaris.readiness.ignore-severe-issues: "true" healthcheck: test: ["CMD", "curl", "http://localhost:8182/q/health"] interval: 10s diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index f06bfad45..930509a20 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -156,8 +156,7 @@ public static void enforceFeatureEnabledOrThrow( List.of( StorageConfigInfo.StorageTypeEnum.S3.name(), StorageConfigInfo.StorageTypeEnum.AZURE.name(), - StorageConfigInfo.StorageTypeEnum.GCS.name(), - StorageConfigInfo.StorageTypeEnum.FILE.name())) + StorageConfigInfo.StorageTypeEnum.GCS.name())) .buildFeatureConfiguration(); public static final FeatureConfiguration CLEANUP_ON_NAMESPACE_DROP = @@ -269,4 +268,23 @@ public static void enforceFeatureEnabledOrThrow( .description("The max number of times to try committing to an Iceberg table") .defaultValue(4) .buildFeatureConfiguration(); + + public static final FeatureConfiguration ALLOW_SPECIFYING_FILE_IO_IMPL = + PolarisConfiguration.builder() + .key("ALLOW_SPECIFYING_FILE_IO_IMPL") + .description( + "Config key for whether to allow setting the FILE_IO_IMPL using catalog properties. " + + "Must only be enabled in dev/test environments, should not be in production systems.") + .defaultValue(false) + .buildFeatureConfiguration(); + + public static final FeatureConfiguration ALLOW_INSECURE_STORAGE_TYPES = + PolarisConfiguration.builder() + .key("ALLOW_INSECURE_STORAGE_TYPES") + .description( + "Allow usage of FileIO implementations that are considered insecure. " + + "Enabling this setting may expose the service to possibly severe security risks!" + + "This should only be set to 'true' for tests!") + .defaultValue(false) + .buildFeatureConfiguration(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/ProductionReadinessCheck.java b/polaris-core/src/main/java/org/apache/polaris/core/config/ProductionReadinessCheck.java index 61d154949..a01e2a076 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/ProductionReadinessCheck.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/ProductionReadinessCheck.java @@ -48,7 +48,11 @@ default boolean ready() { interface Error { static Error of(String message, String offendingProperty) { - return ImmutableError.of(message, offendingProperty); + return ImmutableError.of(message, offendingProperty, false); + } + + static Error ofSevere(String message, String offendingProperty) { + return ImmutableError.of(message, offendingProperty, true); } @Value.Parameter(order = 1) @@ -56,5 +60,8 @@ static Error of(String message, String offendingProperty) { @Value.Parameter(order = 2) String offendingProperty(); + + @Value.Parameter(order = 3) + boolean severe(); } } diff --git a/quarkus/defaults/src/main/resources/application-it.properties b/quarkus/defaults/src/main/resources/application-it.properties index b172114ca..dc660f54a 100644 --- a/quarkus/defaults/src/main/resources/application-it.properties +++ b/quarkus/defaults/src/main/resources/application-it.properties @@ -35,12 +35,14 @@ polaris.features."ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING"=false polaris.features."ALLOW_EXTERNAL_METADATA_FILE_LOCATION"=false polaris.features."ALLOW_OVERLAPPING_CATALOG_URLS"=true polaris.features."ALLOW_SPECIFYING_FILE_IO_IMPL"=true +polaris.features."ALLOW_INSECURE_STORAGE_TYPES"=true polaris.features."ALLOW_WILDCARD_LOCATION"=true polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=true polaris.features."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_it"=true polaris.features."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"] polaris.features."ENABLE_CATALOG_FEDERATION"=true +polaris.readiness.ignore-severe-issues=true polaris.realm-context.realms=POLARIS,OTHER diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index 64f6f2474..b91579011 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -109,7 +109,7 @@ polaris.realm-context.header-name=Polaris-Realm polaris.realm-context.require-header=false polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false -polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] +polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE"] # polaris.features."ENABLE_CATALOG_FEDERATION"=true polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java index 2389537f1..2168fca55 100644 --- a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java @@ -18,6 +18,9 @@ */ package org.apache.polaris.service.quarkus.config; +import static java.lang.String.format; + +import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.event.Observes; import jakarta.enterprise.event.Startup; @@ -27,12 +30,15 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.ProductionReadinessCheck; import org.apache.polaris.core.config.ProductionReadinessCheck.Error; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.auth.AuthenticationRealmConfiguration.TokenBrokerConfiguration.RSAKeyPairConfiguration; import org.apache.polaris.service.auth.AuthenticationRealmConfiguration.TokenBrokerConfiguration.SymmetricKeyConfiguration; import org.apache.polaris.service.auth.AuthenticationType; +import org.apache.polaris.service.catalog.validation.IcebergPropertiesValidation; +import org.apache.polaris.service.config.FeaturesConfiguration; import org.apache.polaris.service.context.DefaultRealmContextResolver; import org.apache.polaris.service.context.RealmContextResolver; import org.apache.polaris.service.context.TestRealmContextResolver; @@ -44,6 +50,7 @@ import org.eclipse.microprofile.config.ConfigValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; @ApplicationScoped public class ProductionReadinessChecks { @@ -57,26 +64,53 @@ public class ProductionReadinessChecks { */ private static final String WARNING_SIGN_UTF_8 = "\u0000\u26A0\uFE0F"; + private static final String SEVERE_SIGN_UTF_8 = "\u0000\uD83D\uDED1"; + /** A simple warning sign displayed when the character set is not UTF-8. */ private static final String WARNING_SIGN_PLAIN = "!!!"; + private static final String SEVERE_SIGN_PLAIN = "***STOP***"; + public void warnOnFailedChecks( - @Observes Startup event, Instance checks) { + @Observes Startup event, + Instance checks, + QuarkusReadinessConfiguration config) { List errors = checks.stream().flatMap(check -> check.getErrors().stream()).toList(); if (!errors.isEmpty()) { - String warning = - Charset.defaultCharset().equals(StandardCharsets.UTF_8) - ? WARNING_SIGN_UTF_8 - : WARNING_SIGN_PLAIN; - LOGGER.warn("{} Production readiness checks failed! Check the warnings below.", warning); + var utf8 = Charset.defaultCharset().equals(StandardCharsets.UTF_8); + var warning = utf8 ? WARNING_SIGN_UTF_8 : WARNING_SIGN_PLAIN; + var severe = utf8 ? SEVERE_SIGN_UTF_8 : SEVERE_SIGN_PLAIN; + var hasSevere = errors.stream().anyMatch(Error::severe); + LOGGER + .makeLoggingEventBuilder(hasSevere ? Level.ERROR : Level.WARN) + .log( + "{} Production readiness checks failed! Check the warnings below.", + hasSevere ? severe : warning); errors.forEach( error -> - LOGGER.warn( - "- {} Offending configuration option: '{}'.", - error.message(), - error.offendingProperty())); - LOGGER.warn( - "Refer to https://polaris.apache.org/in-dev/unreleased/configuring-polaris-for-production for more information."); + LOGGER + .makeLoggingEventBuilder(error.severe() ? Level.ERROR : Level.WARN) + .log( + "- {} {} Offending configuration option: '{}'.", + error.severe() ? severe : warning, + error.message(), + error.offendingProperty())); + LOGGER + .makeLoggingEventBuilder(hasSevere ? Level.ERROR : Level.WARN) + .log( + "Refer to https://polaris.apache.org/in-dev/unreleased/configuring-polaris-for-production for more information."); + + if (hasSevere) { + if (!config.ignoreSevereIssues()) { + throw new IllegalStateException( + "Severe production readiness issues detected, startup aborted!"); + } + LOGGER.warn( + "{} severe production readiness issues detected, but user explicitly requested startup by setting " + + "polaris.readiness.ignore-severe-issues=true and accepts the risk of denial-of-service, " + + "data-loss, corruption and others !", + severe); + } } } @@ -176,4 +210,71 @@ public ProductionReadinessCheck checkPolarisEventListener( private static String authRealmSegment(String realm) { return realm.equals(QuarkusAuthenticationConfiguration.DEFAULT_REALM_KEY) ? "" : realm + "."; } + + @Produces + public ProductionReadinessCheck checkInsecureStorageSettings( + FeaturesConfiguration featureConfiguration) { + var insecure = FeatureConfiguration.ALLOW_INSECURE_STORAGE_TYPES; + + var errors = new ArrayList(); + if (Boolean.parseBoolean(featureConfiguration.defaults().get(insecure.key))) { + errors.add( + Error.ofSevere( + "Must not enable a configuration that exposes known and severe security risks: " + + insecure.description, + format("polaris.features.\"%s\"", insecure.key))); + } + + featureConfiguration + .realmOverrides() + .forEach( + (realmId, overrides) -> { + if (Boolean.parseBoolean(overrides.overrides().get(insecure.key))) { + errors.add( + Error.ofSevere( + "Must not enable a configuration that exposes known and severe security risks: " + + insecure.description, + format( + "polaris.features.realm-overrides.\"%s\".overrides.\"%s\"", + realmId, insecure.key))); + } + }); + + var storageTypes = FeatureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES; + var mapper = new ObjectMapper(); + var defaults = featureConfiguration.parseDefaults(mapper); + var realmOverrides = featureConfiguration.parseRealmOverrides(mapper); + @SuppressWarnings("unchecked") + var supported = (List) defaults.getOrDefault(storageTypes.key, List.of()); + supported.stream() + .filter(n -> !IcebergPropertiesValidation.safeStorageType(n)) + .forEach( + t -> + errors.add( + Error.ofSevere( + format( + "The storage type '%s' is considered insecure and exposes the service to severe security risks!", + t), + format("polaris.features.\"%s\"", storageTypes.key)))); + realmOverrides.forEach( + (realmId, overrides) -> { + @SuppressWarnings("unchecked") + var s = (List) overrides.getOrDefault(storageTypes.key, List.of()); + s.stream() + .filter(n -> !IcebergPropertiesValidation.safeStorageType(n)) + .forEach( + t -> + errors.add( + Error.ofSevere( + format( + "The storage type '%s' is considered insecure and exposes the service to severe security risks!", + t), + format( + "polaris.features.realm-overrides.\"%s\".overrides.\"%s\"", + realmId, storageTypes.key)))); + }); + return errors.isEmpty() + ? ProductionReadinessCheck.OK + : ProductionReadinessCheck.of(errors.toArray(new Error[0])); + } } diff --git a/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusReadinessConfiguration.java b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusReadinessConfiguration.java new file mode 100644 index 000000000..3ff56a7ea --- /dev/null +++ b/quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusReadinessConfiguration.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.config; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.readiness") +public interface QuarkusReadinessConfiguration { + + /** + * Setting this to {@code true} means that Polaris will start up even if severe security risks + * have been detected, accepting the risk of denial-of-service, data-loss, corruption and other + * risks. + */ + @WithDefault("false") + boolean ignoreSevereIssues(); +} diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index 8abb4c041..d3a2b0dba 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -116,7 +116,13 @@ public Map getConfigOverrides() { return Map.of( "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", "true", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", "polaris.features.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"", + "true", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "[\"FILE\",\"S3\"]", + "polaris.readiness.ignore-severe-issues", "true"); } } @@ -226,9 +232,16 @@ public void before(TestInfo testInfo) { Map configMap = Map.of( - "ALLOW_SPECIFYING_FILE_IO_IMPL", true, - "ALLOW_EXTERNAL_METADATA_FILE_LOCATION", true, - "ENABLE_GENERIC_TABLES", true); + "ALLOW_SPECIFYING_FILE_IO_IMPL", + true, + "ALLOW_INSECURE_STORAGE_TYPES", + true, + "ALLOW_EXTERNAL_METADATA_FILE_LOCATION", + true, + "SUPPORTED_CATALOG_STORAGE_TYPES", + List.of("FILE", "S3"), + "ENABLE_GENERIC_TABLES", + true); polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java index f2e929484..8ab23593b 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java @@ -24,6 +24,7 @@ import static org.assertj.core.api.Assertions.assertThat; import jakarta.ws.rs.core.Response; +import java.util.List; import java.util.Map; import java.util.UUID; import java.util.stream.Stream; @@ -73,10 +74,25 @@ private int createTable(TestServices services, String location) { static Stream testTableLocationRestrictions() { Map laxServices = - Map.of("ALLOW_UNSTRUCTURED_TABLE_LOCATION", "true", "ALLOW_TABLE_LOCATION_OVERLAP", "true"); + Map.of( + "ALLOW_UNSTRUCTURED_TABLE_LOCATION", + "true", + "ALLOW_TABLE_LOCATION_OVERLAP", + "true", + "ALLOW_INSECURE_STORAGE_TYPES", + "true", + "SUPPORTED_CATALOG_STORAGE_TYPES", + List.of("FILE", "S3")); Map strictServices = Map.of( - "ALLOW_UNSTRUCTURED_TABLE_LOCATION", "false", "ALLOW_TABLE_LOCATION_OVERLAP", "false"); + "ALLOW_UNSTRUCTURED_TABLE_LOCATION", + "false", + "ALLOW_TABLE_LOCATION_OVERLAP", + "false", + "ALLOW_INSECURE_STORAGE_TYPES", + "true", + "SUPPORTED_CATALOG_STORAGE_TYPES", + List.of("FILE", "S3")); Map laxCatalog = Map.of( ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GetConfigTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GetConfigTest.java index 5e77d6aea..c497cee6a 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GetConfigTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GetConfigTest.java @@ -39,7 +39,16 @@ public class GetConfigTest { @ValueSource(booleans = {true, false}) public void testGetConfig(boolean enableGenericTable) { TestServices services = - TestServices.builder().config(Map.of("ENABLE_GENERIC_TABLES", enableGenericTable)).build(); + TestServices.builder() + .config( + Map.of( + "ALLOW_INSECURE_STORAGE_TYPES", + true, + "SUPPORTED_CATALOG_STORAGE_TYPES", + List.of("FILE", "S3"), + "ENABLE_GENERIC_TABLES", + enableGenericTable)) + .build(); FileStorageConfigInfo fileStorage = FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java index 926e9791e..5c488809d 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerAuthzTest.java @@ -80,21 +80,9 @@ import org.mockito.Mockito; @QuarkusTest -@TestProfile(IcebergCatalogHandlerAuthzTest.Profile.class) +@TestProfile(PolarisAuthzTestBase.Profile.class) public class IcebergCatalogHandlerAuthzTest extends PolarisAuthzTestBase { - public static class Profile extends PolarisAuthzTestBase.Profile { - - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", - "true", - "polaris.features.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"", - "true"); - } - } - private IcebergCatalogHandler newWrapper() { return newWrapper(Set.of()); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index b54d032a8..3bd8f45aa 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -171,14 +171,18 @@ public Map getConfigOverrides() { return Map.of( "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", "true", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", "true", "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\"]", + "[\"FILE\",\"S3\"]", "polaris.features.\"LIST_PAGINATION_ENABLED\"", "true", "polaris.event-listener.type", - "test"); + "test", + "polaris.readiness.ignore-severe-issues", + "true"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 8b87b75a7..0ac72a1b5 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -106,12 +106,16 @@ public Map getConfigOverrides() { "true", "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", "true", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", "true", "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\"]", + "[\"FILE\",\"S3\"]", "polaris.event-listener.type", - "test"); + "test", + "polaris.readiness.ignore-severe-issues", + "true"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java index 5f8c2d028..267fbb386 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java @@ -108,10 +108,14 @@ public Map getConfigOverrides() { return Map.of( "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", "true", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", "true", "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\"]"); + "[\"FILE\"]", + "polaris.readiness.ignore-severe-issues", + "true"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java index 308971c7c..e94f70c63 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java @@ -126,7 +126,11 @@ public Map getConfigOverrides() { "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", "true", "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\"]"); + "[\"FILE\"]", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", + "polaris.readiness.ignore-severe-issues", + "true"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java index 4ae234e55..abeafeb65 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/io/FileIOExceptionsTest.java @@ -27,6 +27,7 @@ import com.google.cloud.storage.StorageException; import jakarta.ws.rs.core.Response; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.Stream; import org.apache.iceberg.Schema; @@ -60,7 +61,15 @@ public class FileIOExceptionsTest { @BeforeAll public static void beforeAll() { - services = TestServices.builder().build(); + services = + TestServices.builder() + .config( + Map.of( + "ALLOW_INSECURE_STORAGE_TYPES", + true, + "SUPPORTED_CATALOG_STORAGE_TYPES", + List.of("FILE", "S3"))) + .build(); ioFactory = (MeasuredFileIOFactory) services.fileIOFactory(); FileStorageConfigInfo storageConfigInfo = diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java index 594efc221..56320c905 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java @@ -47,14 +47,25 @@ public class QuarkusApplicationIntegrationTest extends PolarisApplicationIntegrationTest { public static class Profile implements QuarkusTestProfile { - @Override public Map getConfigOverrides() { return Map.of( - "quarkus.http.limits.max-body-size", "1000000", - "polaris.realm-context.realms", "POLARIS,OTHER", - "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true", - "polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "true"); + "quarkus.http.limits.max-body-size", + "1000000", + "polaris.realm-context.realms", + "POLARIS,OTHER", + "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", + "true", + "polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", + "true", + "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", + "true", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "[\"FILE\",\"S3\"]", + "polaris.readiness.ignore-severe-issues", + "true"); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusPolicyServiceIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusPolicyServiceIntegrationTest.java index a668e92e5..a91f26fc0 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusPolicyServiceIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusPolicyServiceIntegrationTest.java @@ -19,7 +19,27 @@ package org.apache.polaris.service.quarkus.it; import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import java.util.Map; import org.apache.polaris.service.it.test.PolarisPolicyServiceIntegrationTest; @QuarkusTest -public class QuarkusPolicyServiceIntegrationTest extends PolarisPolicyServiceIntegrationTest {} +@TestProfile(QuarkusPolicyServiceIntegrationTest.Profile.class) +public class QuarkusPolicyServiceIntegrationTest extends PolarisPolicyServiceIntegrationTest { + + public static class Profile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", + "true", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "[\"FILE\",\"S3\"]", + "polaris.readiness.ignore-severe-issues", + "true"); + } + } +} diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java index 27e29fd61..b2ddd79b0 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogIntegrationTest.java @@ -32,7 +32,17 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { - return Map.of("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "false"); + return Map.of( + "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", + "true", + "polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", + "false", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "[\"FILE\",\"S3\"]", + "polaris.readiness.ignore-severe-issues", + "true"); } } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java index 3987e94d2..0212f86dc 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java @@ -20,6 +20,7 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; import java.lang.reflect.Field; import java.nio.file.Path; import java.util.Map; @@ -29,6 +30,7 @@ import org.junit.jupiter.api.io.TempDir; @QuarkusTest +@TestProfile(QuarkusRestCatalogViewFileIntegrationTest.Profile.class) public class QuarkusRestCatalogViewFileIntegrationTest extends PolarisRestCatalogViewFileIntegrationTest { @@ -36,7 +38,13 @@ public static class Profile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { - return Map.of("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); + return Map.of( + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", + "true", + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", + "[\"FILE\"]", + "polaris.readiness.ignore-severe-issues", + "true"); } } diff --git a/regtests/docker-compose.yml b/regtests/docker-compose.yml index 24ae7a362..cdb44050d 100644 --- a/regtests/docker-compose.yml +++ b/regtests/docker-compose.yml @@ -34,6 +34,9 @@ services: POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,secret quarkus.log.file.enable: "false" quarkus.otel.sdk.disabled: "true" + polaris.features."ALLOW_INSECURE_STORAGE_TYPES": "true" + polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES": "[\"FILE\",\"S3\",\"GCS\",\"AZURE\"]" + polaris.readiness.ignore-severe-issues: "true" volumes: - ./credentials:/tmp/credentials/ healthcheck: diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 2647c5bc6..ba9df38c9 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -71,7 +71,6 @@ import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.exceptions.UnprocessableEntityException; -import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.io.CloseableGroup; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -92,6 +91,7 @@ import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.config.BehaviorChangeConfiguration; import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.config.PolarisConfiguration; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.NamespaceEntity; @@ -125,6 +125,7 @@ import org.apache.polaris.service.catalog.SupportsNotifications; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.catalog.io.FileIOUtil; +import org.apache.polaris.service.catalog.validation.IcebergPropertiesValidation; import org.apache.polaris.service.events.AfterTableCommitedEvent; import org.apache.polaris.service.events.AfterTableRefreshedEvent; import org.apache.polaris.service.events.AfterViewCommitedEvent; @@ -147,21 +148,6 @@ public class IcebergCatalog extends BaseMetastoreViewCatalog private static final Joiner SLASH = Joiner.on("/"); - // Config key for whether to allow setting the FILE_IO_IMPL using catalog properties. Should - // only be allowed in dev/test environments. - static final String ALLOW_SPECIFYING_FILE_IO_IMPL = "ALLOW_SPECIFYING_FILE_IO_IMPL"; - static final boolean ALLOW_SPECIFYING_FILE_IO_IMPL_DEFAULT = false; - - // Config key for initializing a default "catalogFileIO" that is available either via getIo() - // or for any TableOperations/ViewOperations instantiated, via ops.io() before entity-specific - // FileIO initialization is triggered for any such operations. - // Typically this should only be used in test scenarios where a PolarisIcebergCatalog instance - // is used for both the "client-side" and "server-side" logic instead of being access through - // a REST layer. - static final String INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST = - "INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"; - static final boolean INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST_DEFAULT = false; - public static final Predicate SHOULD_RETRY_REFRESH_PREDICATE = ex -> { // Default arguments from BaseMetastoreTableOperation only stop retries on @@ -258,37 +244,35 @@ public void initialize(String name, Map properties) { properties.getOrDefault(CatalogProperties.WAREHOUSE_LOCATION, ""))); this.defaultBaseLocation = baseLocation.replaceAll("/*$", ""); - Boolean allowSpecifyingFileIoImpl = - getBooleanContextConfiguration( - ALLOW_SPECIFYING_FILE_IO_IMPL, ALLOW_SPECIFYING_FILE_IO_IMPL_DEFAULT); + var storageConfigurationInfo = catalogEntity.getStorageConfigurationInfo(); + ioImplClassName = + IcebergPropertiesValidation.determineFileIOClassName( + callContext, properties, storageConfigurationInfo); - PolarisStorageConfigurationInfo storageConfigurationInfo = - catalogEntity.getStorageConfigurationInfo(); - if (properties.containsKey(CatalogProperties.FILE_IO_IMPL)) { - ioImplClassName = properties.get(CatalogProperties.FILE_IO_IMPL); + if (ioImplClassName == null) { + LOGGER.warn( + "Cannot resolve property '{}' for null storageConfiguration.", + CatalogProperties.FILE_IO_IMPL); + } - if (!Boolean.TRUE.equals(allowSpecifyingFileIoImpl)) { - throw new ValidationException( - "Cannot set property '%s' to '%s' for this catalog.", - CatalogProperties.FILE_IO_IMPL, ioImplClassName); - } + callContext.closeables().addCloseable(this); + this.closeableGroup = new CloseableGroup(); + closeableGroup.addCloseable(metricsReporter()); + closeableGroup.setSuppressCloseFailure(true); + + tableDefaultProperties = + PropertyUtil.propertiesWithPrefix(properties, CatalogProperties.TABLE_DEFAULT_PREFIX); + + if (initializeDefaultCatalogFileioForTest()) { LOGGER.debug( - "Allowing overriding ioImplClassName to {} for storageConfiguration {}", - ioImplClassName, - storageConfigurationInfo); + "Initializing a default catalogFileIO with properties {}", tableDefaultProperties); + this.catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties); + closeableGroup.addCloseable(this.catalogFileIO); } else { - if (storageConfigurationInfo != null) { - ioImplClassName = storageConfigurationInfo.getFileIoImplClassName(); - LOGGER.debug( - "Resolved ioImplClassName {} from storageConfiguration {}", - ioImplClassName, - storageConfigurationInfo); - } else { - LOGGER.warn( - "Cannot resolve property '{}' for null storageConfiguration.", - CatalogProperties.FILE_IO_IMPL); - } + LOGGER.debug("Not initializing default catalogFileIO"); + this.catalogFileIO = null; } + callContext.closeables().addCloseable(this); this.closeableGroup = new CloseableGroup(); closeableGroup.addCloseable(metricsReporter()); @@ -297,11 +281,7 @@ public void initialize(String name, Map properties) { tableDefaultProperties = PropertyUtil.propertiesWithPrefix(properties, CatalogProperties.TABLE_DEFAULT_PREFIX); - Boolean initializeDefaultCatalogFileioForTest = - getBooleanContextConfiguration( - INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST, - INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST_DEFAULT); - if (Boolean.TRUE.equals(initializeDefaultCatalogFileioForTest)) { + if (initializeDefaultCatalogFileioForTest()) { LOGGER.debug( "Initializing a default catalogFileIO with properties {}", tableDefaultProperties); this.catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties); @@ -2565,12 +2545,16 @@ private Boolean getBooleanContextConfiguration(String configKey, boolean default .getConfiguration(callContext.getPolarisCallContext(), configKey, defaultValue); } + private boolean initializeDefaultCatalogFileioForTest() { + var ctx = callContext.getPolarisCallContext(); + return ctx.getConfigurationStore() + .getConfiguration(ctx, INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST); + } + private int getMaxMetadataRefreshRetries() { - return callContext - .getPolarisCallContext() - .getConfigurationStore() - .getConfiguration( - callContext.getPolarisCallContext(), FeatureConfiguration.MAX_METADATA_REFRESH_RETRIES); + var ctx = callContext.getPolarisCallContext(); + return ctx.getConfigurationStore() + .getConfiguration(ctx, FeatureConfiguration.MAX_METADATA_REFRESH_RETRIES); } /** Build a {@link PageToken} from a string and page size. */ @@ -2590,4 +2574,11 @@ private PageToken buildPageToken(@Nullable String tokenString, @Nullable Integer return PageToken.build(tokenString, pageSize); } } + + static final FeatureConfiguration INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST = + PolarisConfiguration.builder() + .key("INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST") + .defaultValue(false) + .description("") + .buildFeatureConfiguration(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index 4c9c527d6..c28fd491a 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -19,6 +19,7 @@ package org.apache.polaris.service.catalog.iceberg; import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; +import static org.apache.polaris.service.catalog.validation.IcebergPropertiesValidation.validateIcebergProperties; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -35,6 +36,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.BadRequestException; @@ -210,6 +212,7 @@ public Response createNamespace( CreateNamespaceRequest createNamespaceRequest, RealmContext realmContext, SecurityContext securityContext) { + validateIcebergProperties(callContext, createNamespaceRequest.properties()); return withCatalog( securityContext, prefix, @@ -301,6 +304,7 @@ public Response updateProperties( UpdateNamespacePropertiesRequest updateNamespacePropertiesRequest, RealmContext realmContext, SecurityContext securityContext) { + validateIcebergProperties(callContext, updateNamespacePropertiesRequest.updates()); Namespace ns = decodeNamespace(namespace); UpdateNamespacePropertiesRequest revisedRequest = UpdateNamespacePropertiesRequest.builder() @@ -335,6 +339,7 @@ public Response createTable( String accessDelegationMode, RealmContext realmContext, SecurityContext securityContext) { + validateIcebergProperties(callContext, createTableRequest.properties()); EnumSet delegationModes = parseAccessDelegationModes(accessDelegationMode); Namespace ns = decodeNamespace(namespace); @@ -506,6 +511,11 @@ public Response updateTable( CommitTableRequest commitTableRequest, RealmContext realmContext, SecurityContext securityContext) { + commitTableRequest.updates().stream() + .filter(MetadataUpdate.SetProperties.class::isInstance) + .map(MetadataUpdate.SetProperties.class::cast) + .forEach(setProperties -> validateIcebergProperties(callContext, setProperties.updated())); + UpdateTableRequest revisedRequest = UpdateTableRequest.create( commitTableRequest.identifier(), @@ -535,6 +545,8 @@ public Response createView( CreateViewRequest createViewRequest, RealmContext realmContext, SecurityContext securityContext) { + validateIcebergProperties(callContext, createViewRequest.properties()); + CreateViewRequest revisedRequest = ImmutableCreateViewRequest.copyOf(createViewRequest) .withProperties( @@ -677,6 +689,12 @@ public Response commitTransaction( CommitTransactionRequest commitTransactionRequest, RealmContext realmContext, SecurityContext securityContext) { + commitTransactionRequest.tableChanges().stream() + .flatMap(updateTableRequest -> updateTableRequest.updates().stream()) + .filter(MetadataUpdate.SetProperties.class::isInstance) + .map(MetadataUpdate.SetProperties.class::cast) + .forEach(setProperties -> validateIcebergProperties(callContext, setProperties.updated())); + CommitTransactionRequest revisedRequest = new CommitTransactionRequest( commitTransactionRequest.tableChanges().stream() diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/validation/IcebergPropertiesValidation.java b/service/common/src/main/java/org/apache/polaris/service/catalog/validation/IcebergPropertiesValidation.java new file mode 100644 index 000000000..c6f1a5dd9 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/validation/IcebergPropertiesValidation.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.validation; + +import static org.apache.polaris.core.config.FeatureConfiguration.ALLOW_INSECURE_STORAGE_TYPES; +import static org.apache.polaris.core.config.FeatureConfiguration.ALLOW_SPECIFYING_FILE_IO_IMPL; +import static org.apache.polaris.core.config.FeatureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES; + +import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; +import java.util.Map; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class IcebergPropertiesValidation { + private static final Logger LOGGER = LoggerFactory.getLogger(IcebergPropertiesValidation.class); + + public static void validateIcebergProperties( + @Nonnull CallContext callContext, @Nonnull Map properties) { + determineFileIOClassName(callContext, properties, null); + } + + public static String determineFileIOClassName( + @Nonnull CallContext callContext, + @Nonnull Map properties, + @Nullable PolarisStorageConfigurationInfo storageConfigurationInfo) { + var ctx = callContext.getPolarisCallContext(); + var configStore = ctx.getConfigurationStore(); + + var ioImpl = properties.get(CatalogProperties.FILE_IO_IMPL); + if (ioImpl != null) { + if (!configStore.getConfiguration(ctx, ALLOW_SPECIFYING_FILE_IO_IMPL)) { + throw new ValidationException( + "Cannot set property '%s' to '%s' for this catalog.", + CatalogProperties.FILE_IO_IMPL, ioImpl); + } + LOGGER.debug( + "Allowing overriding ioImplClassName to {} for storageConfiguration {}", + ioImpl, + storageConfigurationInfo); + } else if (storageConfigurationInfo != null) { + ioImpl = storageConfigurationInfo.getFileIoImplClassName(); + LOGGER.debug( + "Resolved ioImplClassName {} from storageConfiguration {}", + ioImpl, + storageConfigurationInfo); + } + + if (ioImpl != null) { + var storageType = StorageTypeFileIO.fromFileIoImplementation(ioImpl); + if (storageType.validateAllowedStorageType() + && !configStore + .getConfiguration(ctx, SUPPORTED_CATALOG_STORAGE_TYPES) + .contains(storageType.name())) { + throw new ValidationException( + "File IO implementation '%s', as storage type '%s' is not supported", + ioImpl, storageType); + } + + if (!storageType.safe() && !configStore.getConfiguration(ctx, ALLOW_INSECURE_STORAGE_TYPES)) { + throw new ValidationException( + "File IO implementation '%s' (storage type '%s') is considered insecure and must not be used", + ioImpl, storageType); + } + } + + return ioImpl; + } + + public static boolean safeStorageType(String name) { + return StorageTypeFileIO.valueOf(name).safe(); + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/validation/StorageTypeFileIO.java b/service/common/src/main/java/org/apache/polaris/service/catalog/validation/StorageTypeFileIO.java new file mode 100644 index 000000000..55684ab2d --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/validation/StorageTypeFileIO.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.validation; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; + +enum StorageTypeFileIO { + S3("org.apache.iceberg.aws.s3.S3FileIO", true), + + GCS("org.apache.iceberg.gcp.gcs.GCSFileIO", true), + + AZURE("org.apache.iceberg.azure.adlsv2.ADLSFileIO", true), + + FILE("org.apache.iceberg.hadoop.HadoopFileIO", false), + + // Iceberg tests + IN_MEMORY("org.apache.iceberg.inmemory.InMemoryFileIO", false, false), + ; + + private final String fileIoImplementation; + private final boolean safe; + private final boolean validateAllowedStorageType; + + StorageTypeFileIO(String fileIoImplementation, boolean safe) { + this(fileIoImplementation, safe, true); + } + + StorageTypeFileIO(String fileIoImplementation, boolean safe, boolean validateAllowedStorageType) { + this.fileIoImplementation = fileIoImplementation; + this.safe = safe; + this.validateAllowedStorageType = validateAllowedStorageType; + } + + boolean safe() { + return safe; + } + + boolean validateAllowedStorageType() { + return validateAllowedStorageType; + } + + static StorageTypeFileIO fromFileIoImplementation(String fileIoImplementation) { + var type = FILE_TO_TO_STORAGE_TYPE.get(fileIoImplementation); + if (type == null) { + throw new IllegalArgumentException("Unknown FileIO implementation: " + fileIoImplementation); + } + return type; + } + + private static final Map FILE_TO_TO_STORAGE_TYPE; + + static { + var map = new HashMap(); + for (var st : PolarisStorageConfigurationInfo.StorageType.values()) { + // Ensure all storage types are included in this enum + valueOf(st.name()); + } + for (var value : StorageTypeFileIO.values()) { + if (value.validateAllowedStorageType()) { + // Ensure that the storage type in this enum has a corresponding value + PolarisStorageConfigurationInfo.StorageType.valueOf(value.name()); + } + map.put(value.fileIoImplementation, value); + } + FILE_TO_TO_STORAGE_TYPE = Collections.unmodifiableMap(map); + } +} diff --git a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java index 30afc5c09..3dae85660 100644 --- a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java @@ -121,7 +121,14 @@ FileIO loadFileIOInternal( testServices = TestServices.builder() - .config(Map.of("ALLOW_SPECIFYING_FILE_IO_IMPL", true)) + .config( + Map.of( + "ALLOW_SPECIFYING_FILE_IO_IMPL", + true, + "ALLOW_INSECURE_STORAGE_TYPES", + true, + "SUPPORTED_CATALOG_STORAGE_TYPES", + List.of("FILE", "S3"))) .realmContext(realmContext) .stsClient(stsClient) .fileIOFactorySupplier(fileIOFactorySupplier)