Require explict user-consent to enable HadoopFileIO

Using `HadoopFileIO` in Polaris can enable "hidden features" that users are likely not aware of. This change requires users to manually update the configuration to be able to use `HadoopFileIO` in way that highlights the consequences of enabling it.

This PR updates Polaris in multiple ways:
* The default of `SUPPORTED_CATALOG_STORAGE_TYPES` is changed to not include the `FILE` storage type.
* Respect the `ALLOW_SPECIFYING_FILE_IO_IMPL` configuration on namespaces, tables and views to prevent setting an `io-impl` value for anything but one of the configured, supported storage-types.
* Unify validation code in a new class `IcebergPropertiesValidation`.
* Using `FILE` or `HadoopFileIO` now _also_ requires the explicit configuration `ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS=true`.
* Added production readiness checks that trigger when `ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS` is `true` or `SUPPORTED_CATALOG_STORAGE_TYPES` contains `FILE` (defaults and per-realm).
* The two new readiness checks are considered _severe_. Severe readiness-errors prevent the server from starting up - unless the user explicitly configured `polaris.readiness.ignore-security-issues=true`.

Log messages and configuration options explicitly use "clear" phrases highlighting the consequences.

With these changes it is intentionally extremely difficult to start Polaris with HadoopFileIO. People who work around all these safety nets must have realized that what they are doing.

A lot of the test code relies on `FILE`/`HadoopFileIO`, those tests got all the configurations to let those tests continue to work as they are, bypassing the added security safeguards.

Author: snazy

plugins/spark/v3.5/regtests/docker-compose.yml

@@ -28,6 +28,9 @@ services:
       POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,root,secret
       quarkus.log.file.enable: "false"
       quarkus.otel.sdk.disabled: "true"
+      polaris.features.defaults."ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS": "true"
+      polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES": "[\"FILE\",\"S3\",\"GCS\",\"AZURE\"]"
+      polaris.readiness.ignore-security-issues: "true"
     healthcheck:
       test: ["CMD", "curl", "http://localhost:8182/q/health"]
       interval: 10s

polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java

@@ -146,8 +146,7 @@ public static void enforceFeatureEnabledOrThrow(
               List.of(
                   StorageConfigInfo.StorageTypeEnum.S3.name(),
                   StorageConfigInfo.StorageTypeEnum.AZURE.name(),
-                  StorageConfigInfo.StorageTypeEnum.GCS.name(),
-                  StorageConfigInfo.StorageTypeEnum.FILE.name()))
+                  StorageConfigInfo.StorageTypeEnum.GCS.name()))
           .buildFeatureConfiguration();
 
   public static final FeatureConfiguration<Boolean> CLEANUP_ON_NAMESPACE_DROP =
@@ -234,4 +233,38 @@ public static void enforceFeatureEnabledOrThrow(
           .description("If true, the policy-store endpoints are enabled")
           .defaultValue(true)
           .buildFeatureConfiguration();
+
+  public static final FeatureConfiguration<Boolean> ALLOW_SPECIFYING_FILE_IO_IMPL =
+      PolarisConfiguration.<Boolean>builder()
+          .key("ALLOW_SPECIFYING_FILE_IO_IMPL")
+          .description(
+              "Config key for whether to allow setting the FILE_IO_IMPL using catalog properties. "
+                  + "Must only be enabled in dev/test environments, never in production systems.")
+          .defaultValue(false)
+          .buildFeatureConfiguration();
+
+  public static final FeatureConfiguration<Boolean>
+      ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS =
+          PolarisConfiguration.<Boolean>builder()
+              .key("ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS")
+              .description(
+                  "Allow usage of FileIO implementations that are considered insecure. "
+                      + "Enabling this setting exposes the service to SEVERE security risks, including "
+                      + "denial of service, corruption, data loss and more!"
+                      + "This must NEVER be set to 'true' for anything except tests! ")
+              .defaultValue(false)
+              .buildFeatureConfiguration();
+
+  public static final FeatureConfiguration<Boolean> INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST =
+      PolarisConfiguration.<Boolean>builder()
+          .key("INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST")
+          .description(
+              "Config key for initializing a default \"catalogFileIO\" that is available either via "
+                  + "getIo() or for any TableOperations/ViewOperations instantiated, via ops.io() "
+                  + "before entity-specific FileIO initialization is triggered for any such operations. "
+                  + "Typically this should only be used in test scenarios where a PolarisIcebergCatalog "
+                  + "instance is used for both the \"client-side\" and \"server-side\" logic instead of "
+                  + "being access through a REST layer.")
+          .defaultValue(false)
+          .buildFeatureConfiguration();
 }

polaris-core/src/main/java/org/apache/polaris/core/config/ProductionReadinessCheck.java

@@ -48,13 +48,20 @@ default boolean ready() {
   interface Error {
 
     static Error of(String message, String offendingProperty) {
-      return ImmutableError.of(message, offendingProperty);
+      return ImmutableError.of(message, offendingProperty, false);
+    }
+
+    static Error ofSevere(String message, String offendingProperty) {
+      return ImmutableError.of(message, offendingProperty, true);
     }
 
     @Value.Parameter(order = 1)
     String message();
 
     @Value.Parameter(order = 2)
     String offendingProperty();
+
+    @Value.Parameter(order = 3)
+    boolean severe();
   }
 }

quarkus/defaults/src/main/resources/application-it.properties

@@ -35,12 +35,14 @@ polaris.features.defaults."ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING"=false
 polaris.features.defaults."ALLOW_EXTERNAL_METADATA_FILE_LOCATION"=false
 polaris.features.defaults."ALLOW_OVERLAPPING_CATALOG_URLS"=true
 polaris.features.defaults."ALLOW_SPECIFYING_FILE_IO_IMPL"=true
+polaris.features.defaults."ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS"=true
 polaris.features.defaults."ALLOW_WILDCARD_LOCATION"=true
 polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=true
 polaris.features.defaults."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_it"=true
 polaris.features.defaults."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true
 polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"]
 polaris.features.defaults."ENABLE_CATALOG_FEDERATION"=true
+polaris.readiness.ignore-security-issues=true
 
 polaris.realm-context.realms=POLARIS,OTHER
 

quarkus/defaults/src/main/resources/application.properties

@@ -109,7 +109,7 @@ polaris.realm-context.header-name=Polaris-Realm
 polaris.realm-context.require-header=false
 
 polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false
-polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"]
+polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE"]
 # polaris.features.defaults."ENABLE_CATALOG_FEDERATION"=true
 
 # realm overrides

quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/ProductionReadinessChecks.java

@@ -18,6 +18,9 @@
  */
 package org.apache.polaris.service.quarkus.config;
 
+import static java.lang.String.format;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.enterprise.event.Observes;
 import jakarta.enterprise.event.Startup;
@@ -27,12 +30,15 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
+import org.apache.polaris.core.config.FeatureConfiguration;
 import org.apache.polaris.core.config.ProductionReadinessCheck;
 import org.apache.polaris.core.config.ProductionReadinessCheck.Error;
 import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
 import org.apache.polaris.service.auth.AuthenticationRealmConfiguration.TokenBrokerConfiguration.RSAKeyPairConfiguration;
 import org.apache.polaris.service.auth.AuthenticationRealmConfiguration.TokenBrokerConfiguration.SymmetricKeyConfiguration;
 import org.apache.polaris.service.auth.AuthenticationType;
+import org.apache.polaris.service.catalog.validation.IcebergPropertiesValidation;
+import org.apache.polaris.service.config.FeaturesConfiguration;
 import org.apache.polaris.service.context.DefaultRealmContextResolver;
 import org.apache.polaris.service.context.RealmContextResolver;
 import org.apache.polaris.service.context.TestRealmContextResolver;
@@ -42,6 +48,7 @@
 import org.eclipse.microprofile.config.ConfigValue;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
 
 @ApplicationScoped
 public class ProductionReadinessChecks {
@@ -55,26 +62,53 @@ public class ProductionReadinessChecks {
    */
   private static final String WARNING_SIGN_UTF_8 = "\u0000\u26A0\uFE0F";
 
+  private static final String SEVERE_SIGN_UTF_8 = "\u0000\uD83D\uDED1";
+
   /** A simple warning sign displayed when the character set is not UTF-8. */
   private static final String WARNING_SIGN_PLAIN = "!!!";
 
+  private static final String SEVERE_SIGN_PLAIN = "***STOP***";
+
   public void warnOnFailedChecks(
-      @Observes Startup event, Instance<ProductionReadinessCheck> checks) {
+      @Observes Startup event,
+      Instance<ProductionReadinessCheck> checks,
+      QuarkusReadinessConfiguration config) {
     List<Error> errors = checks.stream().flatMap(check -> check.getErrors().stream()).toList();
     if (!errors.isEmpty()) {
-      String warning =
-          Charset.defaultCharset().equals(StandardCharsets.UTF_8)
-              ? WARNING_SIGN_UTF_8
-              : WARNING_SIGN_PLAIN;
-      LOGGER.warn("{} Production readiness checks failed! Check the warnings below.", warning);
+      var utf8 = Charset.defaultCharset().equals(StandardCharsets.UTF_8);
+      var warning = utf8 ? WARNING_SIGN_UTF_8 : WARNING_SIGN_PLAIN;
+      var severe = utf8 ? SEVERE_SIGN_UTF_8 : SEVERE_SIGN_PLAIN;
+      var hasSevere = errors.stream().anyMatch(Error::severe);
+      LOGGER
+          .makeLoggingEventBuilder(hasSevere ? Level.ERROR : Level.WARN)
+          .log(
+              "{} Production readiness checks failed! Check the warnings below.",
+              hasSevere ? severe : warning);
       errors.forEach(
           error ->
-              LOGGER.warn(
-                  "- {} Offending configuration option: '{}'.",
-                  error.message(),
-                  error.offendingProperty()));
-      LOGGER.warn(
-          "Refer to https://polaris.apache.org/in-dev/unreleased/configuring-polaris-for-production for more information.");
+              LOGGER
+                  .makeLoggingEventBuilder(error.severe() ? Level.ERROR : Level.WARN)
+                  .log(
+                      "- {} {} Offending configuration option: '{}'.",
+                      error.severe() ? severe : warning,
+                      error.message(),
+                      error.offendingProperty()));
+      LOGGER
+          .makeLoggingEventBuilder(hasSevere ? Level.ERROR : Level.WARN)
+          .log(
+              "Refer to https://polaris.apache.org/in-dev/unreleased/configuring-polaris-for-production for more information.");
+
+      if (hasSevere) {
+        if (!config.ignoreSecurityIssues()) {
+          throw new IllegalStateException(
+              "Severe production readiness issues detected, startup aborted!");
+        }
+        LOGGER.error(
+            "{} severe production readiness issues detected, but user explicitly requested startup by setting "
+                + "polaris.readiness.ignore-security-issues=true and accepts the risk of denial-of-service, "
+                + "data-loss, corruption and others !",
+            severe);
+      }
     }
   }
 
@@ -164,4 +198,71 @@ public ProductionReadinessCheck checkRealmResolver(Config config, RealmContextRe
   private static String authRealmSegment(String realm) {
     return realm.equals(QuarkusAuthenticationConfiguration.DEFAULT_REALM_KEY) ? "" : realm + ".";
   }
+
+  @Produces
+  public ProductionReadinessCheck checkInsecureStorageSettings(
+      FeaturesConfiguration featureConfiguration) {
+    var insecure = FeatureConfiguration.ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS;
+
+    var errors = new ArrayList<Error>();
+    if (Boolean.parseBoolean(featureConfiguration.defaults().get(insecure.key))) {
+      errors.add(
+          Error.ofSevere(
+              "Must not enable a configuration that exposes known and severe security risks: "
+                  + insecure.description,
+              format("polaris.features.defaults.\"%s\"", insecure.key)));
+    }
+
+    featureConfiguration
+        .realmOverrides()
+        .forEach(
+            (realmId, overrides) -> {
+              if (Boolean.parseBoolean(overrides.overrides().get(insecure.key))) {
+                errors.add(
+                    Error.ofSevere(
+                        "Must not enable a configuration that exposes known and severe security risks: "
+                            + insecure.description,
+                        format(
+                            "polaris.features.realm-overrides.\"%s\".overrides.\"%s\"",
+                            realmId, insecure.key)));
+              }
+            });
+
+    var storageTypes = FeatureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES;
+    var mapper = new ObjectMapper();
+    var defaults = featureConfiguration.parseDefaults(mapper);
+    var realmOverrides = featureConfiguration.parseRealmOverrides(mapper);
+    @SuppressWarnings("unchecked")
+    var supported = (List<String>) defaults.getOrDefault(storageTypes.key, List.of());
+    supported.stream()
+        .filter(n -> !IcebergPropertiesValidation.safeStorageType(n))
+        .forEach(
+            t ->
+                errors.add(
+                    Error.ofSevere(
+                        format(
+                            "The storage type '%s' is considered insecure and to expose the service to severe security ricks!",
+                            t),
+                        format("polaris.features.defaults.\"%s\"", storageTypes.key))));
+    realmOverrides.forEach(
+        (realmId, overrides) -> {
+          @SuppressWarnings("unchecked")
+          var s = (List<String>) overrides.getOrDefault(storageTypes.key, List.of());
+          s.stream()
+              .filter(n -> !IcebergPropertiesValidation.safeStorageType(n))
+              .forEach(
+                  t ->
+                      errors.add(
+                          Error.ofSevere(
+                              format(
+                                  "The storage type '%s' is considered insecure and to expose the service to severe security ricks!",
+                                  t),
+                              format(
+                                  "polaris.features.realm-overrides.\"%s\".overrides.\"%s\"",
+                                  realmId, storageTypes.key))));
+        });
+    return errors.isEmpty()
+        ? ProductionReadinessCheck.OK
+        : ProductionReadinessCheck.of(errors.toArray(new Error[0]));
+  }
 }

quarkus/service/src/main/java/org/apache/polaris/service/quarkus/config/QuarkusReadinessConfiguration.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.quarkus.config;
+
+import io.quarkus.runtime.annotations.StaticInitSafe;
+import io.smallrye.config.ConfigMapping;
+import io.smallrye.config.WithDefault;
+
+@StaticInitSafe
+@ConfigMapping(prefix = "polaris.readiness")
+public interface QuarkusReadinessConfiguration {
+
+  /**
+   * Setting this to {@code true} means that Polaris will start up even if severe security risks
+   * have been detected, accepting the risk of denial-of-service, data-loss, corruption and other
+   * risks.
+   */
+  @WithDefault("false")
+  boolean ignoreSecurityIssues();
+}

quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java

@@ -113,7 +113,13 @@ public Map<String, String> getConfigOverrides() {
       return Map.of(
           "polaris.features.defaults.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
           "true",
+          "polaris.features.defaults.\"ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS\"",
+          "true",
           "polaris.features.defaults.\"ALLOW_EXTERNAL_METADATA_FILE_LOCATION\"",
+          "true",
+          "polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
+          "[\"FILE\",\"S3\"]",
+          "polaris.readiness.ignore-security-issues",
           "true");
     }
   }
@@ -220,9 +226,16 @@ public void before(TestInfo testInfo) {
 
     Map<String, Object> configMap =
         Map.of(
-            "ALLOW_SPECIFYING_FILE_IO_IMPL", true,
-            "ALLOW_EXTERNAL_METADATA_FILE_LOCATION", true,
-            "ENABLE_GENERIC_TABLES", true);
+            "ALLOW_SPECIFYING_FILE_IO_IMPL",
+            true,
+            "ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS",
+            true,
+            "ALLOW_EXTERNAL_METADATA_FILE_LOCATION",
+            true,
+            "SUPPORTED_CATALOG_STORAGE_TYPES",
+            List.of("FILE", "S3"),
+            "ENABLE_GENERIC_TABLES",
+            true);
     polarisContext =
         new PolarisCallContext(
             managerFactory.getOrCreateSessionSupplier(realmContext).get(),

quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java

@@ -24,6 +24,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 
 import jakarta.ws.rs.core.Response;
+import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import java.util.stream.Stream;
@@ -73,10 +74,25 @@ private int createTable(TestServices services, String location) {
 
   static Stream<Arguments> testTableLocationRestrictions() {
     Map<String, Object> laxServices =
-        Map.of("ALLOW_UNSTRUCTURED_TABLE_LOCATION", "true", "ALLOW_TABLE_LOCATION_OVERLAP", "true");
+        Map.of(
+            "ALLOW_UNSTRUCTURED_TABLE_LOCATION",
+            "true",
+            "ALLOW_TABLE_LOCATION_OVERLAP",
+            "true",
+            "ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS",
+            "true",
+            "SUPPORTED_CATALOG_STORAGE_TYPES",
+            List.of("FILE", "S3"));
     Map<String, Object> strictServices =
         Map.of(
-            "ALLOW_UNSTRUCTURED_TABLE_LOCATION", "false", "ALLOW_TABLE_LOCATION_OVERLAP", "false");
+            "ALLOW_UNSTRUCTURED_TABLE_LOCATION",
+            "false",
+            "ALLOW_TABLE_LOCATION_OVERLAP",
+            "false",
+            "ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS",
+            "true",
+            "SUPPORTED_CATALOG_STORAGE_TYPES",
+            List.of("FILE", "S3"));
     Map<String, Object> laxCatalog =
         Map.of(
             ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(),

quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java

@@ -106,10 +106,14 @@ public Map<String, String> getConfigOverrides() {
       return Map.of(
           "polaris.features.defaults.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"",
           "true",
+          "polaris.features.defaults.\"ALLOW_INSECURE_STORAGE_TYPES_ACCEPTING_SECURITY_RISKS\"",
+          "true",
           "polaris.features.defaults.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"",
           "true",
           "polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"",
-          "[\"FILE\"]");
+          "[\"FILE\"]",
+          "polaris.readiness.ignore-security-issues",
+          "true");
     }
   }