diff --git a/build.gradle b/build.gradle index 34b797553d..d5c998c84f 100644 --- a/build.gradle +++ b/build.gradle @@ -571,6 +571,10 @@ allprojects { } integrationTestImplementation 'org.slf4j:slf4j-api:2.0.12' integrationTestImplementation 'com.selectivem.collections:special-collections-complete:1.4.0' + + integrationTestImplementation ('com.jayway.jsonpath:json-path:2.9.0') { + exclude(group: 'net.minidev', module: 'json-smart') + } } } diff --git a/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java b/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java index e27845d95b..34857ea2a7 100644 --- a/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java +++ b/src/integrationTest/java/org/opensearch/security/http/ServiceAccountAuthenticationTest.java @@ -20,11 +20,11 @@ import org.junit.Test; import org.junit.runner.RunWith; -import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestIndex; import static org.opensearch.security.support.ConfigConstants.SECURITY_RESTAPI_ROLES_ENABLED; import static org.opensearch.security.support.ConfigConstants.SECURITY_SYSTEM_INDICES_ENABLED_KEY; diff --git a/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java index adc0b212f5..94a10233e0 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java @@ -19,12 +19,12 @@ import org.opensearch.script.mustache.MustacheModulePlugin; import org.opensearch.script.mustache.RenderSearchTemplateAction; -import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestIndex; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; diff --git a/src/integrationTest/java/org/opensearch/security/privileges/dlsfls/FlsFmIntegrationTests.java b/src/integrationTest/java/org/opensearch/security/privileges/dlsfls/FlsFmIntegrationTests.java index 068972f9e2..abee5eb844 100644 --- a/src/integrationTest/java/org/opensearch/security/privileges/dlsfls/FlsFmIntegrationTests.java +++ b/src/integrationTest/java/org/opensearch/security/privileges/dlsfls/FlsFmIntegrationTests.java @@ -28,12 +28,12 @@ import org.bouncycastle.util.encoders.Hex; import org.opensearch.plugin.mapper.MapperSizePlugin; -import org.opensearch.test.framework.TestData; -import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.cluster.ClusterManager; import org.opensearch.test.framework.cluster.LocalCluster; import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestData; +import org.opensearch.test.framework.data.TestIndex; import com.rfksystems.blake2b.Blake2b; diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/ClusterConfig.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/ClusterConfig.java new file mode 100644 index 0000000000..2d3a1627e0 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/ClusterConfig.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.privileges.int_tests; + +import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; + +import org.opensearch.test.framework.cluster.LocalCluster; + +/** + * This is one of the test parameter dimensions used by the *Authorization*IntTests test suites. + * The test suites run on different cluster configurations; the possible cluster configurations are defined here. + */ +public enum ClusterConfig { + LEGACY_PRIVILEGES_EVALUATION( + "legacy", + c -> c.doNotFailOnForbidden(true).nodeSettings(Map.of("plugins.security.system_indices.enabled", true)), + true, + false, + false + ), + LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION( + "legacy_system_index_perm", + c -> c.doNotFailOnForbidden(true) + .nodeSettings( + Map.of("plugins.security.system_indices.enabled", true, "plugins.security.system_indices.permission.enabled", true) + ), + true, + true, + false + ); + + final String name; + final Function clusterConfiguration; + final boolean legacyPrivilegeEvaluation; + final boolean systemIndexPrivilegeEnabled; + final boolean allowsEmptyResultSets; + + private LocalCluster cluster; + + ClusterConfig( + String name, + Function clusterConfiguration, + boolean legacyPrivilegeEvaluation, + boolean systemIndexPrivilegeEnabled, + boolean allowsEmptyResultSets + ) { + this.name = name; + this.clusterConfiguration = clusterConfiguration; + this.legacyPrivilegeEvaluation = legacyPrivilegeEvaluation; + this.systemIndexPrivilegeEnabled = systemIndexPrivilegeEnabled; + this.allowsEmptyResultSets = allowsEmptyResultSets; + } + + LocalCluster cluster(Supplier clusterBuilder) { + if (cluster == null) { + cluster = this.clusterConfiguration.apply(clusterBuilder.get()).build(); + cluster.before(); + } + return cluster; + } + + void shutdown() { + if (cluster != null) { + try { + cluster.close(); + } catch (Exception e) {} + cluster = null; + } + } + + @Override + public String toString() { + return name; + } +} diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java new file mode 100644 index 0000000000..10a107d057 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadOnlyIntTests.java @@ -0,0 +1,882 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.privileges.int_tests; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestComponentTemplate; +import org.opensearch.test.framework.data.TestDataStream; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.data.TestIndexTemplate; +import org.opensearch.test.framework.matcher.RestIndexMatchers; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.data.TestIndex.openSearchSecurityConfigIndex; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimited; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; +import static org.opensearch.test.framework.matcher.RestMatchers.isNotFound; +import static org.opensearch.test.framework.matcher.RestMatchers.isOk; + +/** + * This class defines a huge test matrix for index related access controls. This class is especially for read-only operations on data streams. + * It uses the following dimensions: + *
    + *
  • ClusterConfig: At the moment, we test without and with system index permission enabled. New semantics will follow later.
    • + *
  • TestSecurityConfig.User: We have quite a few of different users with different privileges configurations.
    • + *
  • The test methods represent different operations with different options that are tested
    • + *
+ * To cope with the huge space of tests, this class uses test oracles to verify the result of the operations. + * These are defined with the "indexMatcher()" method of TestSecurityConfig.User. See there and the class IndexApiMatchers. + */ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class DataStreamAuthorizationReadOnlyIntTests { + + // ------------------------------------------------------------------------------------------------------- + // Test data streams and indices used by this test suite. Indices are usually initially created; the only + // exception is ds_ax, which is referred to in tests, but which does not exist on purpose. + // ------------------------------------------------------------------------------------------------------- + + static TestDataStream ds_a1 = TestDataStream.name("ds_a1").documentCount(100).rolloverAfter(10).seed(1).build(); + static TestDataStream ds_a2 = TestDataStream.name("ds_a2").documentCount(110).rolloverAfter(10).seed(2).build(); + static TestDataStream ds_a3 = TestDataStream.name("ds_a3").documentCount(120).rolloverAfter(10).seed(3).build(); + static TestDataStream ds_ax = TestDataStream.name("ds_ax").build(); // Not existing data stream + static TestDataStream ds_b1 = TestDataStream.name("ds_b1").documentCount(51).rolloverAfter(10).seed(4).build(); + static TestDataStream ds_b2 = TestDataStream.name("ds_b2").documentCount(52).rolloverAfter(10).seed(5).build(); + static TestDataStream ds_b3 = TestDataStream.name("ds_b3").documentCount(53).rolloverAfter(10).seed(6).build(); + static TestIndex index_c1 = TestIndex.name("index_c1").documentCount(5).seed(7).build(); + + static final List ALL_INDICES = List.of( + ds_a1, + ds_a2, + ds_a3, + ds_b1, + ds_b2, + ds_b3, + index_c1, + openSearchSecurityConfigIndex() + ); + + static final List ALL_INDICES_EXCEPT_SYSTEM_INDICES = List.of( + ds_a1, + ds_a2, + ds_a3, + ds_b1, + ds_b2, + ds_b3, + index_c1 + ); + + static final List ALL_DATA_STREAMS = List.of(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3); + + /** + * This key identifies assertion reference data for index search/read permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey READ = new TestSecurityConfig.User.MetadataKey<>( + "read", + RestIndexMatchers.IndexMatcher.class + ); + + // ------------------------------------------------------------------------------------------------------- + // Test users with which the tests will be executed; the users need to be added to the list USERS below + // The users have two redundant versions or privilege configuration, which needs to be kept in sync: + // - The standard role configuration defined with .roles() + // - IndexMatchers which act as test oracles, defined with the indexMatcher() methods + // ------------------------------------------------------------------------------------------------------- + + /** + * A simple user that can read from ds_a* + */ + static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// + .description("ds_a*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// + .indexPermissions( + "read", + "indices_monitor", + "indices:admin/analyze", + "indices:admin/data_stream/get", + "indices:monitor/data_stream/stats" + ) + .on("ds_a*") + )// + .reference(READ, limitedTo(ds_a1, ds_a2, ds_a3, ds_ax)); + + /** + * A simple user that can read from ds_b* + */ + static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// + .description("ds_b*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// + .indexPermissions( + "read", + "indices_monitor", + "indices:admin/analyze", + "indices:admin/data_stream/get", + "indices:monitor/data_stream/stats" + ) + .on("ds_b*") + )// + .reference(READ, limitedTo(ds_b1, ds_b2, ds_b3)); + + /** + * A simple user that can read from ds_b1 + */ + static TestSecurityConfig.User LIMITED_USER_B1 = new TestSecurityConfig.User("limited_user_B1")// + .description("ds_b1")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// + .indexPermissions( + "read", + "indices_monitor", + "indices:admin/analyze", + "indices:admin/data_stream/get", + "indices:monitor/data_stream/stats" + ) + .on("ds_b1") + )// + .reference(READ, limitedTo(ds_b1)); + + /** + * This user has no privileges for indices that are used in this test. But they have privileges for other indices. + * This allows them to use actions like _search and receive empty result sets. + */ + static TestSecurityConfig.User LIMITED_USER_OTHER_PRIVILEGES = new TestSecurityConfig.User("limited_user_other_index_privileges")// + .description("no privileges for existing indices")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// + .indexPermissions( + "read", + "indices_monitor", + "indices:admin/analyze", + "indices:admin/data_stream/get", + "indices:monitor/data_stream/stats" + ) + .on("ds_does_not_exist_*") + )// + .reference(READ, limitedToNone()); + + /** + * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index + * restrictions and similar things. + */ + static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// + .description("unlimited")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor")// + .indexPermissions("*") + .on("*")// + )// + .reference(READ, unlimited()); + + /** + * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. + * This serves as a base for comparison with the default behavior. + */ + static TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// + .description("super unlimited (admin cert)")// + .adminCertUser()// + .reference(READ, unlimitedIncludingOpenSearchSecurityIndex()); + + static List USERS = List.of( + LIMITED_USER_A, + LIMITED_USER_B, + LIMITED_USER_B1, + LIMITED_USER_OTHER_PRIVILEGES, + UNLIMITED_USER, + SUPER_UNLIMITED_USER + ); + + static LocalCluster.Builder clusterBuilder() { + return new LocalCluster.Builder().singleNode() + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USERS)// + .indexTemplates(new TestIndexTemplate("ds_test", "ds_*").dataStream().composedOf(TestComponentTemplate.DATA_STREAM_MINIMAL))// + .dataStreams(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3)// + .indices(index_c1); + } + + @AfterClass + public static void stopClusters() { + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + clusterConfig.shutdown(); + } + } + + final TestSecurityConfig.User user; + final LocalCluster cluster; + final ClusterConfig clusterConfig; + + @Test + public void search_noPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000"); + assertThat( + httpResponse, + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_noPattern_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000&expand_wildcards=none"); + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // The dnfof implementation has the effect that the expand_wildcards=none option is disregarded + // Additionally, the dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_noPattern_allowNoIndicesFalse() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000&allow_no_indices=false"); + + assertThat( + httpResponse, + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isNotFound() : isForbidden()) + ); + } + } + + @Test + public void search_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_all_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000&expand_wildcards=none"); + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // The dnfof implementation has the effect that the expand_wildcards=none option is disregarded + // Additionally, the dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*/_search?size=1000"); + + assertThat( + httpResponse, + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_staticNames_noIgnoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1/_search?size=1000"); + // With dnfof data streams with incomplete privileges will be replaced by their member indices + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) + ); + } + } + + @Test + public void search_staticNames_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1/_search?size=1000&ignore_unavailable=true"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_staticIndicies_negation_backingIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1,-.ds-ds_b1*/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_indexPattern_minus() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*,-ds_b2,-ds_b3/_search?size=1000"); + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + // does not handle the expression ds_a*,ds_b*,-ds_b2,-ds_b3 in a way that excludes the data streams. See + // search_indexPattern_minus_backingIndices for an alternative. + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // The IndexResolverReplacer fails to interpret the minus patterns and falls back to interpreting the given index names + // literally + // In the logs, this then looks like this: + // | indices:data/read/search | + // -ds_b2| MISSING | + // -ds_b3| MISSING | + // ds_b* | MISSING | + // ds_a* | MISSING | + // This has the effect that granted privileges using wildcards might work, but granted privileges without wildcards won't + // work + if (user == LIMITED_USER_B1) { + // No wildcard in the index pattern + assertThat(httpResponse, isForbidden()); + } else { + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + } + + @Test + public void search_indexPattern_minus_backingIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*,-.ds-ds_b2*,-.ds-ds_b3*/_search?size=1000"); + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + + // dnfof has the effect that the index expression is interpreted differently and that ds_b2 and ds_b3 get included + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_indexPattern_nonExistingIndex_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get( + "ds_a*,ds_b*,xxx_non_existing/_search?size=1000&ignore_unavailable=true" + ); + + // The presence of a non existing index has the effect that the other patterns are not resolved by IndexResolverReplacer + // This causes a few more 403 errors where the granted index patterns do not use wildcards + + if (user == LIMITED_USER_B1) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_indexPattern_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get( + "ds_a*,ds_b*/_search?size=1000&expand_wildcards=none&ignore_unavailable=true" + ); + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // dnfof makes the expand_wildcards=none option ineffective + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_nonExisting_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist/_search?size=1000"); + + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isNotFound()); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/search]")); + } + } + } + + @Test + public void search_nonExisting_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist*/_search?size=1000"); + + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } + } + + @Test + public void search_termsAggregation_index() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("_search", """ + { + "size": 0, + "aggs": { + "indices": { + "terms": { + "field": "_index", + "size": 1000 + } + } + } + }"""); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("aggregations.indices.buckets[*].key") + .reducedBy(user.reference(READ)) + .whenEmpty(isOk()) + ); + } else { + // Users without full privileges will not see hidden indices here; thus on a cluster with only data streams, the result is + // often just empty + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("aggregations.indices.buckets[*].key")); + } + } + } + + @Test + public void msearch_staticIndices() throws Exception { + String msearchBody = """ + {"index": "ds_b1"} + {"size": 10, "query": {"bool":{"must":{"match_all":{}}}}} + {"index": "ds_b2"} + {"size": 10, "query": {"bool":{"must":{"match_all":{}}}}} + """; + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("_msearch", msearchBody); + assertThat( + httpResponse, + containsExactly(ds_b1, ds_b2).at("responses[*].hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isOk()) + ); + } + } + + @Test + public void index_stats_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_stats"); + assertThat( + httpResponse, + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("indices.keys()") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void index_stats_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_b*/_stats"); + assertThat( + httpResponse, + containsExactly(ds_b1, ds_b2, ds_b3).at("indices.keys()") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void getDataStream_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream"); + // The legacy mode does not support dnfof for indices:admin/data_stream/get + assertThat( + httpResponse, + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.reference(READ)) + ); + } + } + + @Test + public void getDataStream_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/*"); + // The legacy mode does not support dnfof for indices:admin/data_stream/get + assertThat( + httpResponse, + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.reference(READ)) + ); + } + } + + @Test + public void getDataStream_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_a*"); + // The legacy mode does not support dnfof for indices:admin/data_stream/get + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.reference(READ)) + ); + } + } + + @Test + public void getDataStream_pattern_negation() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_*,-ds_b*"); + // The legacy mode does not support dnfof for indices:admin/data_stream/get + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.reference(READ)) + ); + } + } + + @Test + public void getDataStream_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_a1,ds_a2"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2).at("$.data_streams[*].name").butForbiddenIfIncomplete(user.reference(READ)) + ); + } + } + + @Test + public void getDataStreamStats_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/_stats"); + // The legacy mode does not support dnfof for indices:monitor/data_stream/stats + assertThat( + httpResponse, + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.reference(READ)) + ); + } + } + + @Test + public void getDataStreamStats_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/*/_stats"); + // The legacy mode does not support dnfof for indices:monitor/data_stream/stats + assertThat( + httpResponse, + containsExactly(ALL_DATA_STREAMS).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.reference(READ)) + ); + } + } + + @Test + public void getDataStreamStats_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_a*/_stats"); + // The legacy mode does not support dnfof for indices:monitor/data_stream/stats + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.reference(READ)) + ); + } + } + + @Test + public void getDataStreamStats_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_data_stream/ds_a1,ds_a2/_stats"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2).at("$.data_streams[*].data_stream").butForbiddenIfIncomplete(user.reference(READ)) + ); + } + } + + @Test + public void resolve_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/*"); + assertThat( + httpResponse, + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("$.*[*].name") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void resolve_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/ds_a*,ds_b*"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("$.*[*].name") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void field_caps_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(ALL_INDICES_EXCEPT_SYSTEM_INDICES).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void field_caps_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*/_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void field_caps_staticIndices_noIgnoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1/_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("indices").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) + ); + + } + } + + @Test + public void field_caps_staticIndices_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1/_field_caps?fields=*&ignore_unavailable=true"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void field_caps_nonExisting_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist/_field_caps?fields=*"); + + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isNotFound()); + } else { + assertThat(httpResponse.getStatusCode(), is(403)); + } + } + } + + @Test + public void field_caps_nonExisting_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist*/_field_caps?fields=*"); + + assertThat(httpResponse, containsExactly().at("indices").whenEmpty(isOk())); + } + } + + @Test + public void field_caps_indexPattern_minus() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*,-ds_b2,-ds_b3/_field_caps?fields=*"); + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + // OpenSearch does not handle the expression ds_a*,ds_b*,-ds_b2,-ds_b3 in a way that excludes the data streams. See + // field_caps_indexPattern_minus_backingIndices for an alternative. + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + if (user == LIMITED_USER_B1) { + // No wildcard in the index pattern + assertThat(httpResponse, isForbidden()); + } else { + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + } + + @Test + public void field_caps_indexPattern_minus_backingIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a*,ds_b*,-.ds-ds_b2*,-.ds-ds_b3*/_field_caps?fields=*"); + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // dnfof has the effect that the index expression is interpreted differently and that ds_b2 and ds_b3 get included + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_a3, ds_b1, ds_b2, ds_b3).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void field_caps_staticIndices_negation_backingIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("ds_a1,ds_a2,ds_b1,-.ds-ds_b1*/_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(ds_a1, ds_a2, ds_b1).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") + public static Collection params() { + List result = new ArrayList<>(); + + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + for (TestSecurityConfig.User user : USERS) { + result.add(new Object[] { clusterConfig, user, user.getDescription() }); + } + } + return result; + } + + public DataStreamAuthorizationReadOnlyIntTests(ClusterConfig clusterConfig, TestSecurityConfig.User user, String description) + throws Exception { + this.user = user; + this.cluster = clusterConfig.cluster(DataStreamAuthorizationReadOnlyIntTests::clusterBuilder); + this.clusterConfig = clusterConfig; + } +} diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java new file mode 100644 index 0000000000..11a341f726 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/DataStreamAuthorizationReadWriteIntTests.java @@ -0,0 +1,634 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.privileges.int_tests; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import javax.annotation.concurrent.NotThreadSafe; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.google.common.collect.ImmutableList; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.action.admin.indices.refresh.RefreshRequest; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import org.opensearch.test.framework.data.TestComponentTemplate; +import org.opensearch.test.framework.data.TestDataStream; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.data.TestIndexTemplate; +import org.opensearch.test.framework.matcher.RestIndexMatchers; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.cluster.TestRestClient.json; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimited; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestMatchers.isBadRequest; +import static org.opensearch.test.framework.matcher.RestMatchers.isCreated; +import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; +import static org.opensearch.test.framework.matcher.RestMatchers.isOk; +import static org.junit.Assert.assertEquals; + +/** + * This class defines a huge test matrix for index related access controls. This class is especially for read/write operations on data streams. + * It uses the following dimensions: + *
    + *
  • ClusterConfig: At the moment, we test without and with system index permission enabled. New semantics will follow later.
    • + *
  • TestSecurityConfig.User: We have quite a few of different users with different privileges configurations.
    • + *
  • The test methods represent different operations with different options that are tested
    • + *
+ * To cope with the huge space of tests, this class uses test oracles to verify the result of the operations. + * These are defined with the "indexMatcher()" method of TestSecurityConfig.User. See there and the class IndexApiMatchers. + */ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +@NotThreadSafe +public class DataStreamAuthorizationReadWriteIntTests { + + // ------------------------------------------------------------------------------------------------------- + // Test indices used by this test suite. We use the following naming scheme: + // - index_*r*, ds_*r*: This test will not write to this index or data stream + // - index_*w*, ds_*w*: This test can write to this index or data stream; the test won't delete and recreate it + // - index_*wx*, ds_*wx*: The index is not initially created; the test can create it on demand and delete it again + // ------------------------------------------------------------------------------------------------------- + + static TestDataStream ds_ar1 = TestDataStream.name("ds_ar1").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_ar2 = TestDataStream.name("ds_ar2").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_aw1 = TestDataStream.name("ds_aw1").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_aw2 = TestDataStream.name("ds_aw2").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_br1 = TestDataStream.name("ds_br1").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_br2 = TestDataStream.name("ds_br2").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_bw1 = TestDataStream.name("ds_bw1").documentCount(22).rolloverAfter(10).build(); + static TestDataStream ds_bw2 = TestDataStream.name("ds_bw2").documentCount(22).rolloverAfter(10).build(); + static TestIndex index_cr1 = TestIndex.name("index_cr1").documentCount(10).build(); + static TestIndex index_cw1 = TestIndex.name("index_cw1").documentCount(10).build(); + static TestDataStream ds_hidden = TestDataStream.name("ds_hidden").documentCount(10).rolloverAfter(3).seed(8).build(); + + static TestDataStream ds_bwx1 = TestDataStream.name("ds_bwx1").documentCount(0).build(); // not initially created + static TestDataStream ds_bwx2 = TestDataStream.name("ds_bwx2").documentCount(0).build(); // not initially created + + /** + * This key identifies assertion reference data for index search/read permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey READ = new TestSecurityConfig.User.MetadataKey<>( + "read", + RestIndexMatchers.IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for index write permissions of individual users. This does + * not include index creation permissions. + */ + static final TestSecurityConfig.User.MetadataKey WRITE = new TestSecurityConfig.User.MetadataKey<>( + "write", + RestIndexMatchers.IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for create index permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey CREATE_DATA_STREAM = + new TestSecurityConfig.User.MetadataKey<>("create_data_stream", RestIndexMatchers.IndexMatcher.class); + + /** + * This key identifies assertion reference data for manage index permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey MANAGE_DATA_STREAM = + new TestSecurityConfig.User.MetadataKey<>("manage_data_stream", RestIndexMatchers.IndexMatcher.class); + + // ------------------------------------------------------------------------------------------------------- + // Test users with which the tests will be executed; the users need to be added to the list USERS below + // The users have two redundant versions or privilege configuration, which needs to be kept in sync: + // - The standard role configuration defined with .roles() + // - IndexMatchers which act as test oracles, defined with the indexMatcher() methods + // ------------------------------------------------------------------------------------------------------- + + /** + * A simple user that can read from ds_a* and write to ds_aw*; the user as no privileges to create or manage data streams + */ + static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// + .description("ds_a*")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("ds_a*")// + .indexPermissions("write") + .on("ds_aw*") + )// + .reference(READ, limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// + .reference(WRITE, limitedTo(ds_aw1, ds_aw2))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); + + /** + * A simple user that can read from ds_b* and write to ds_bw*; the user as no privileges to create or manage data streams + */ + static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// + .description("ds_b*")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("ds_b*")// + .indexPermissions("write") + .on("ds_bw*") + )// + .reference(READ, limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); + + /** + * A simple user that can read from ds_b* and write to ds_bw*; the user as no privileges to create or manage data streams. + * Additionally, they can read from ds_a* + */ + static TestSecurityConfig.User LIMITED_USER_B_READ_ONLY_A = new TestSecurityConfig.User("limited_user_B_read_only_A")// + .description("ds_b*; read only on ds_a*")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("ds_a*", "ds_b*")// + .indexPermissions("write") + .on("ds_bw*") + )// + .reference(READ, limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2, ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); + + /** + * This is an artificial user - in the sense that in real life it would likely not exist this way. + * It has privileges to write on ds_b*, but privileges for indices:admin/mapping/auto_put on all data streams. + * The reason is that some indexing operations are two phase - first auto put, then indexing. To be able to test both + * phases, we need which user which always allows the first phase to pass. + */ + static TestSecurityConfig.User LIMITED_USER_B_AUTO_PUT_ON_ALL = new TestSecurityConfig.User("limited_user_B_auto_put_on_all")// + .description("ds_b* with full auto put")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("ds_b*")// + .indexPermissions("write") + .on("ds_bw*")// + .indexPermissions("indices:admin/mapping/auto_put") + .on("*") + )// + .reference(READ, limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); + + /** + * A simple user that can read from ds_b* and write to ds_bw*; they can also create data streams with the name ds_bw* + */ + static TestSecurityConfig.User LIMITED_USER_B_CREATE_DS = new TestSecurityConfig.User("limited_user_B_create_ds")// + .description("ds_b* with create ds privs")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("ds_b*")// + .indexPermissions("write") + .on("ds_bw*")// + .indexPermissions("indices:admin/data_stream/create") + .on("ds_bw*") + )// + .reference(READ, limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(MANAGE_DATA_STREAM, limitedToNone()); + + /** + * A simple user that can read from ds_b* and write to ds_bw*; they can also create and manage data streams with the name ds_bw* + */ + static TestSecurityConfig.User LIMITED_USER_B_MANAGE_DS = new TestSecurityConfig.User("limited_user_B_manage_ds")// + .description("ds_b* with manage privs")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("ds_b*")// + .indexPermissions("write") + .on("ds_bw*")// + .indexPermissions("manage") + .on("ds_bw*") + )// + .reference(READ, limitedTo(ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(MANAGE_DATA_STREAM, limitedTo(ds_bw1, ds_bw2, ds_bwx1, ds_bwx2)); + + /** + * A user that can read from ds_a* and ds_b* and write/create/manage ds_aw*, ds_bw* + */ + static TestSecurityConfig.User LIMITED_USER_AB_MANAGE_INDEX = new TestSecurityConfig.User("limited_user_AB_manage_index")// + .description("ds_a*, ds_b* with manage index privs")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("ds_a*", "ds_b*")// + .indexPermissions("write") + .on("ds_aw*", "ds_bw*")// + .indexPermissions("manage") + .on("ds_aw*", "ds_bw*") + )// + .reference(READ, limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2, ds_br1, ds_br2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(WRITE, limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(CREATE_DATA_STREAM, limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2))// + .reference(MANAGE_DATA_STREAM, limitedTo(ds_aw1, ds_aw2, ds_bw1, ds_bw2, ds_bwx1, ds_bwx2)); + + /** + * A simple user that can read from index_c* + */ + static TestSecurityConfig.User LIMITED_USER_C = new TestSecurityConfig.User("limited_user_C")// + .description("index_c*")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("index_c*")// + .indexPermissions("write") + .on("index_cw*") + )// + .reference(READ, limitedTo(index_cr1, index_cw1))// + .reference(WRITE, limitedTo(index_cw1))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); + + /** + * A simple user that can read all indices and data streams, but cannot write anything + */ + static TestSecurityConfig.User LIMITED_READ_ONLY_ALL = new TestSecurityConfig.User("limited_read_only_all")// + .description("read/only on *")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read") + .on("*") + )// + .reference(READ, unlimited())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); + + /** + * A simple user that can read from ds_a*, but cannot write anything + */ + static TestSecurityConfig.User LIMITED_READ_ONLY_A = new TestSecurityConfig.User("limited_read_only_A")// + .description("read/only on ds_a*")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read") + .on("ds_a*") + )// + .reference(READ, limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// + .reference(WRITE, limitedToNone())// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); + + /** + * A simple test user that only has index privileges for indices that are not used by this test. + */ + static TestSecurityConfig.User LIMITED_USER_OTHER_PRIVILEGES = new TestSecurityConfig.User("limited_user_other_privileges")// + .description("no privileges for existing indices")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("crud", "indices_monitor") + .on("ds_does_not_exist_*") + )// + .reference(READ, limitedToNone())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); + + /** + * A simple test user that has no index privileges at all. + */ + static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// + .description("no index privileges")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + )// + .reference(READ, limitedToNone())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); + + /** + * This user has only privileges on backing indices for data streams, but not on the data streams themselves + */ + static TestSecurityConfig.User LIMITED_USER_PERMISSIONS_ON_BACKING_INDICES = new TestSecurityConfig.User( + "limited_user_permissions_on_backing_indices" + )// + .description("ds_a* on backing indices")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on(".ds-ds_a*")// + .indexPermissions("write") + .on(".ds-ds_aw*") + )// + .reference(READ, limitedTo(ds_ar1, ds_ar2, ds_aw1, ds_aw2))// + .reference(WRITE, limitedTo(ds_aw1, ds_aw2))// + .reference(CREATE_DATA_STREAM, limitedToNone())// + .reference(MANAGE_DATA_STREAM, limitedToNone()); + + /** + * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index + * restrictions and similar things. + */ + static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// + .description("unlimited")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("*") + .on("*") + )// + .reference(READ, unlimited())// + .reference(WRITE, unlimited())// + .reference(CREATE_DATA_STREAM, unlimited())// + .reference(MANAGE_DATA_STREAM, unlimited()); + + /** + * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. + * This serves as a base for comparison with the default behavior. + */ + static TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// + .description("super unlimited (admin cert)")// + .adminCertUser()// + .reference(READ, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(WRITE, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(CREATE_DATA_STREAM, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(MANAGE_DATA_STREAM, unlimitedIncludingOpenSearchSecurityIndex()); + + static List USERS = ImmutableList.of( + LIMITED_USER_A, + LIMITED_USER_B, + LIMITED_USER_B_READ_ONLY_A, + LIMITED_USER_B_AUTO_PUT_ON_ALL, + LIMITED_USER_B_CREATE_DS, + LIMITED_USER_B_MANAGE_DS, + LIMITED_USER_AB_MANAGE_INDEX, + LIMITED_USER_C, + LIMITED_READ_ONLY_ALL, + LIMITED_READ_ONLY_A, + LIMITED_USER_OTHER_PRIVILEGES, + LIMITED_USER_NONE, + LIMITED_USER_PERMISSIONS_ON_BACKING_INDICES, + UNLIMITED_USER, + SUPER_UNLIMITED_USER + ); + + static LocalCluster.Builder clusterBuilder() { + return new LocalCluster.Builder().singleNode() + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USERS)// + .indexTemplates(new TestIndexTemplate("ds_test", "ds_*").dataStream().composedOf(TestComponentTemplate.DATA_STREAM_MINIMAL))// + .indices(index_cr1, index_cw1)// + .dataStreams(ds_ar1, ds_ar2, ds_aw1, ds_aw2, ds_br1, ds_br2, ds_bw1, ds_bw2, ds_hidden)// + .plugin(IndexAuthorizationReadOnlyIntTests.SystemIndexTestPlugin.class); + } + + @AfterClass + public static void stopClusters() { + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + clusterConfig.shutdown(); + } + } + + final TestSecurityConfig.User user; + final LocalCluster cluster; + final ClusterConfig clusterConfig; + + @Test + public void createDocument() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.post("ds_bw1/_doc/", json("a", 1, "@timestamp", Instant.now().toString())); + assertThat(httpResponse, containsExactly(ds_bw1).at("_index").reducedBy(user.reference(WRITE)).whenEmpty(isForbidden())); + } + } + + @Test + public void deleteByQuery_indexPattern() throws Exception { + String testName = "deleteByQuery_indexPattern"; + + try (TestRestClient restClient = cluster.getRestClient(user)) { + try (TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + // Init test data + HttpResponse httpResponse = adminRestClient.put( + "ds_bw1/_create/put_delete_delete_by_query_b1?refresh=true", + json("test", testName, "delete_by_query_test_delete", "yes", "@timestamp", Instant.now().toString()) + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "ds_bw1/_create/put_delete_delete_by_query_b2?refresh=true", + json("test", testName, "delete_by_query_test_delete", "no", "@timestamp", Instant.now().toString()) + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "ds_aw1/_create/put_delete_delete_by_query_a1?refresh=true", + json("test", testName, "delete_by_query_test_delete", "yes", "@timestamp", Instant.now().toString()) + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "ds_aw1/_create/put_delete_delete_by_query_a2?refresh=true", + json("test", testName, "delete_by_query_test_delete", "no", "@timestamp", Instant.now().toString()) + ); + assertThat(httpResponse, isCreated()); + } + + HttpResponse httpResponse = restClient.postJson("ds_aw*,ds_bw*/_delete_by_query?wait_for_completion=true", """ + { + "query": { + "term": { + "delete_by_query_test_delete": "yes" + } + } + } + """); + + if (clusterConfig.legacyPrivilegeEvaluation) { + // dnfof is not applicable to indices:data/write/delete/byquery, so we need privileges for all indices + if (user.reference(WRITE).coversAll(ds_aw1, ds_aw2, ds_bw1, ds_bw2)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } else { + if (user != LIMITED_USER_NONE && user != LIMITED_READ_ONLY_ALL && user != LIMITED_READ_ONLY_A) { + assertThat(httpResponse, isOk()); + int expectedDeleteCount = containsExactly(ds_aw1, ds_bw1).at("_index").reducedBy(user.reference(WRITE)).size(); + assertEquals(httpResponse.getBody(), expectedDeleteCount, httpResponse.bodyAsMap().get("deleted")); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + deleteTestDocs(testName, "ds_aw*,ds_bw*"); + } + } + + @Test + public void putDocument_bulk() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson("_bulk", """ + { "create": { "_index": "ds_aw1", "_id": "d1" } } + { "a": 1, "test": "putDocument_bulk", "@timestamp": "2025-09-15T12:00:00Z" } + { "create": { "_index": "ds_bw1", "_id": "d1" } } + { "b": 1, "test": "putDocument_bulk", "@timestamp": "2025-09-15T12:00:01Z" } + """); + + if (user == LIMITED_USER_PERMISSIONS_ON_BACKING_INDICES) { + // IndexResolverReplacer won't resolve data stream names to member index names, because it does not + // specify the includeDataStream option and thus just stumbles over an IndexNotFoundException + // Thus, in contrast to aliases, privileges on backing index names won't work + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("items[*].create[?(@.result == 'created')]._index")); + } else if (user != LIMITED_USER_NONE) { + assertThat( + httpResponse, + containsExactly(ds_aw1, ds_bw1).at("items[*].create[?(@.result == 'created')]._index") + .reducedBy(user.reference(WRITE)) + .whenEmpty(isOk()) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + deleteTestDocs("putDocument_bulk", "ds_aw*,ds_bw*"); + } + } + + @Test + public void createDataStream() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.put("_data_stream/ds_bwx1"); + + if (containsExactly(ds_bwx1).reducedBy(user.reference(CREATE_DATA_STREAM)).isEmpty()) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat(httpResponse, isOk()); + } + } finally { + delete(ds_bwx1); + } + } + + @Test + public void putDataStream() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson("ds_bwx1/", "{}"); + + if (user == UNLIMITED_USER + || user == SUPER_UNLIMITED_USER + || user == LIMITED_USER_B_MANAGE_DS + || user == LIMITED_USER_AB_MANAGE_INDEX) { + // This will fail because we try to create an index under a name of a data stream + assertThat(httpResponse, isBadRequest()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(ds_bwx1); + } + } + + @Test + public void deleteDataStream() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(ds_bwx1); + + HttpResponse httpResponse = restClient.delete("_data_stream/ds_bwx1"); + + if (user.reference(MANAGE_DATA_STREAM).isEmpty()) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat(httpResponse, isOk()); + } + } finally { + delete(ds_bwx1); + } + } + + @After + public void refresh() { + cluster.getInternalNodeClient().admin().indices().refresh(new RefreshRequest("*")).actionGet(); + } + + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") + public static Collection params() { + List result = new ArrayList<>(); + + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + for (TestSecurityConfig.User user : USERS) { + result.add(new Object[] { clusterConfig, user, user.getDescription() }); + } + } + return result; + } + + public DataStreamAuthorizationReadWriteIntTests(ClusterConfig clusterConfig, TestSecurityConfig.User user, String description) + throws Exception { + this.user = user; + this.cluster = clusterConfig.cluster(DataStreamAuthorizationReadWriteIntTests::clusterBuilder); + this.clusterConfig = clusterConfig; + } + + private void createInitialTestObjects(TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + TestIndexOrAliasOrDatastream.createInitialTestObjects(cluster, testIndexOrAliasOrDatastreamArray); + } + + private void delete(TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + TestIndexOrAliasOrDatastream.delete(cluster, testIndexOrAliasOrDatastreamArray); + } + + private void deleteTestDocs(String testName, String indices) { + try (TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + adminRestClient.post(indices + "/_refresh"); + adminRestClient.postJson(indices + "/_delete_by_query?refresh=true&wait_for_completion=true", """ + { + "query": { + "term": { + "test.keyword": "%s" + } + } + } + """.formatted(testName)); + } catch (Exception e) { + throw new RuntimeException("Error while cleaning up test docs", e); + } + } + +} diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java new file mode 100644 index 0000000000..5182f6fb53 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadOnlyIntTests.java @@ -0,0 +1,1912 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.privileges.int_tests; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.stream.Stream; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.google.common.collect.ImmutableList; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.common.settings.Settings; +import org.opensearch.indices.SystemIndexDescriptor; +import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SystemIndexPlugin; +import org.opensearch.script.mustache.MustacheModulePlugin; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestAlias; +import org.opensearch.test.framework.data.TestData; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.matcher.RestIndexMatchers; + +import static java.util.stream.Collectors.joining; +import static org.apache.commons.lang3.StringEscapeUtils.escapeJson; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.cluster.TestRestClient.json; +import static org.opensearch.test.framework.data.TestIndex.openSearchSecurityConfigIndex; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.IndexMatcher; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestMatchers.isBadRequest; +import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; +import static org.opensearch.test.framework.matcher.RestMatchers.isNotFound; +import static org.opensearch.test.framework.matcher.RestMatchers.isOk; +import static org.junit.Assert.assertTrue; + +/** + * This class defines a huge test matrix for index related access controls. This class is especially for read-only operations on indices and aliases. + * It uses the following dimensions: + *
    + *
  • ClusterConfig: At the moment, we test without and with system index permission enabled. New semantics will follow later.
    • + *
  • TestSecurityConfig.User: We have quite a few of different users with different privileges configurations.
    • + *
  • The test methods represent different operations with different options that are tested
    • + *
+ * To cope with the huge space of tests, this class uses test oracles to verify the result of the operations. + * These are defined with the "indexMatcher()" method of TestSecurityConfig.User. See there and the class IndexApiMatchers. + */ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class IndexAuthorizationReadOnlyIntTests { + + // ------------------------------------------------------------------------------------------------------- + // Test indices used by this test suite. Indices are usually initially created; the only exception is + // index_ax, which is referred to in tests, but which does not exist on purpose. + // ------------------------------------------------------------------------------------------------------- + + static final TestIndex index_a1 = TestIndex.name("index_a1").documentCount(100).seed(1).build(); + static final TestIndex index_a2 = TestIndex.name("index_a2").documentCount(110).seed(2).build(); + static final TestIndex index_a3 = TestIndex.name("index_a3").documentCount(120).seed(3).build(); + static final TestIndex index_ax = TestIndex.name("index_ax").build(); // Not existing index + static final TestIndex index_b1 = TestIndex.name("index_b1").documentCount(51).seed(4).build(); + static final TestIndex index_b2 = TestIndex.name("index_b2").documentCount(52).seed(5).build(); + static final TestIndex index_b3 = TestIndex.name("index_b3").documentCount(53).seed(6).build(); + static final TestIndex index_c1 = TestIndex.name("index_c1").documentCount(5).seed(7).build(); + static final TestIndex index_hidden = TestIndex.name("index_hidden").hidden().documentCount(1).seed(8).build(); + static final TestIndex index_hidden_dot = TestIndex.name(".index_hidden_dot").hidden().documentCount(1).seed(8).build(); + static final TestIndex system_index_plugin = TestIndex.name(".system_index_plugin").hidden().documentCount(1).seed(8).build(); + + static final TestAlias alias_ab1 = new TestAlias("alias_ab1").on(index_a1, index_a2, index_a3, index_b1); + static final TestAlias alias_c1 = new TestAlias("alias_c1").on(index_c1); + static final TestAlias alias_with_system_index = new TestAlias(".alias_with_system_index").hidden().on(system_index_plugin); + + static final List ALL_INDICES_EXCEPT_SYSTEM_INDICES = List.of( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot + ); + + static final List ALL_INDICES = List.of( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin, + openSearchSecurityConfigIndex() + ); + + static final List ALL_INDICES_AND_ALIASES = List.of( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + alias_ab1, + alias_c1, + index_hidden, + index_hidden_dot, + system_index_plugin, + alias_with_system_index, + openSearchSecurityConfigIndex() + ); + + static final List ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES = List.of( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + alias_ab1, + alias_c1, + index_hidden, + index_hidden_dot + ); + + /** + * This key identifies assertion reference data for index search/read permissions of individual users. + * This reference data generally applies for both legacy privilege evaluation and new privilege evaluation. + * There might be however deviations in some cases; then the READ_NEXT_GEN matcher can be used. + */ + static final TestSecurityConfig.User.MetadataKey READ = new TestSecurityConfig.User.MetadataKey<>( + "read", + IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for index search/read permissions of individual users for the new privilege evaluation + */ + static final TestSecurityConfig.User.MetadataKey READ_NEXT_GEN = new TestSecurityConfig.User.MetadataKey<>( + "read_nextgen", + IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for operations getting alias meta data of individual users + */ + static final TestSecurityConfig.User.MetadataKey GET_ALIAS = new TestSecurityConfig.User.MetadataKey<>( + "get_alias", + IndexMatcher.class + ); + + // ------------------------------------------------------------------------------------------------------- + // Test users with which the tests will be executed; the users need to be added to the list USERS below + // The users have two redundant versions or privilege configuration, which needs to be kept in sync: + // - The standard role configuration defined with .roles() + // - IndexMatchers which act as test oracles, defined with the indexMatcher() methods + // ------------------------------------------------------------------------------------------------------- + + /** + * A simple user that can read from index_a* + */ + static final TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// + .description("index_a*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze") + .on("index_a*") + )// + .reference(READ, limitedTo(index_a1, index_a2, index_a3, index_ax))// + .reference(GET_ALIAS, limitedToNone()); + + /** + * A simple user that can read from index_b* + */ + static final TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// + .description("index_b*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze") + .on("index_b*") + )// + .reference(READ, limitedTo(index_b1, index_b2, index_b3))// + .reference(GET_ALIAS, limitedToNone()); + + /** + * A simple user that can read only from index_b1 + */ + static final TestSecurityConfig.User LIMITED_USER_B1 = new TestSecurityConfig.User("limited_user_B1")// + .description("index_b1")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze") + .on("index_b1") + )// + .reference(READ, limitedTo(index_b1))// + .reference(GET_ALIAS, limitedToNone()); + + /** + * A simple user that can read from index_c* + */ + static final TestSecurityConfig.User LIMITED_USER_C = new TestSecurityConfig.User("limited_user_C")// + .description("index_c*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze") + .on("index_c*") + )// + .reference(READ, limitedTo(index_c1, alias_c1))// + .reference(GET_ALIAS, limitedToNone()); + + /** + * A user that has read privileges for alias_ab1*; these privileges are inherited to the member indices. + * The user has no directly defined privileges on indices. + */ + static final TestSecurityConfig.User LIMITED_USER_ALIAS_AB1 = new TestSecurityConfig.User("limited_user_alias_AB1")// + .description("alias_ab1")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze", "indices:admin/aliases/get") + .on("alias_ab1*") + )// + .reference(READ, limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1))// + .reference(GET_ALIAS, limitedTo(index_a1, index_a2, index_a3, index_b1, alias_ab1)); + + /** + * A user that has read privileges for alias_c1; these privileges are inherited to the member indices. + * The user has no directly defined privileges on indices. + */ + static final TestSecurityConfig.User LIMITED_USER_ALIAS_C1 = new TestSecurityConfig.User("limited_user_alias_C1")// + .description("alias_c1")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze", "indices:admin/aliases/get") + .on("alias_c1") + )// + .reference(READ, limitedTo(index_c1, alias_c1))// + .reference(GET_ALIAS, limitedTo(index_c1, alias_c1)); + /** + * Same as LIMITED_USER_A with the addition of read privileges for index_hidden* and .index_hidden* + */ + static final TestSecurityConfig.User LIMITED_USER_A_HIDDEN = new TestSecurityConfig.User("limited_user_A_hidden")// + .description("index_a*, index_hidden*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze") + .on("index_a*", "index_hidden*", ".index_hidden*") + )// + .reference(READ, limitedTo(index_a1, index_a2, index_a3, index_ax, index_hidden, index_hidden_dot))// + .reference(GET_ALIAS, limitedToNone()); + + /** + * Same as LIMITED_USER_C with the addition of read privileges for ".system_index_plugin"; they also have the + * explicit privilege "system:admin/system_index" that allows them accessing this index. + */ + static final TestSecurityConfig.User LIMITED_USER_C_WITH_SYSTEM_INDICES = new TestSecurityConfig.User( + "limited_user_C_with_system_indices" + )// + .description("index_c*, .system_index_plugin")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("read", "indices_monitor", "indices:admin/analyze", "indices:admin/aliases/get") + .on("index_c*", "alias_c1")// + .indexPermissions( + "read", + "indices_monitor", + "indices:admin/analyze", + "indices:admin/aliases/get", + "system:admin/system_index" + ) + .on(".system_index_plugin") + )// + .reference(READ, limitedTo(index_c1, alias_c1, system_index_plugin, alias_with_system_index))// + .reference(GET_ALIAS, limitedTo(index_c1, alias_c1, system_index_plugin, alias_with_system_index)); + + /** + * This user has no privileges for indices that are used in this test. But they have privileges for other indices. + * This allows them to use actions like _search and receive empty result sets. + *

+ * Compare with LIMITED_USER_NONE, which has no search privileges and will only receive 403 errors. + */ + static final TestSecurityConfig.User LIMITED_USER_OTHER_PRIVILEGES = new TestSecurityConfig.User("limited_user_other_index_privileges")// + .description("no privileges for tested indices")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + .indexPermissions("crud", "indices_monitor", "indices:admin/analyze") + .on("index_does_not_exist_*") + )// + .reference(READ, limitedToNone())// + .reference(GET_ALIAS, limitedToNone()); + + /** + * This user has no index read privileges at all. + */ + static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// + .description("no index privileges")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + )// + .reference(READ, limitedToNone())// + .reference(GET_ALIAS, limitedToNone()); + + /** + * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index + * restrictions and similar things. + */ + static final TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// + .description("unlimited")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("*") + .indexPermissions("*") + .on("*")// + + )// + .reference(READ, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_ax))// + .reference(GET_ALIAS, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_ax)); + + /** + * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. + * This serves as a base for comparison with the default behavior. + */ + static final TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// + .description("super unlimited (admin cert)")// + .adminCertUser()// + .reference(READ, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(GET_ALIAS, unlimitedIncludingOpenSearchSecurityIndex()); + + static final List USERS = ImmutableList.of( + LIMITED_USER_A, + LIMITED_USER_B, + LIMITED_USER_B1, + LIMITED_USER_C, + LIMITED_USER_ALIAS_AB1, + LIMITED_USER_ALIAS_C1, + LIMITED_USER_A_HIDDEN, + LIMITED_USER_C_WITH_SYSTEM_INDICES, + LIMITED_USER_OTHER_PRIVILEGES, + LIMITED_USER_NONE, + UNLIMITED_USER, + SUPER_UNLIMITED_USER + ); + + static LocalCluster.Builder clusterBuilder() { + return new LocalCluster.Builder().singleNode() + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USERS)// + .indices( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + )// + .aliases(alias_ab1, alias_c1, alias_with_system_index)// + .plugin(SystemIndexTestPlugin.class, MustacheModulePlugin.class); + } + + @AfterClass + public static void stopClusters() { + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + clusterConfig.shutdown(); + } + } + + final TestSecurityConfig.User user; + final LocalCluster cluster; + final ClusterConfig clusterConfig; + + @Test + public void search_noPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // The dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_noPattern_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000&expand_wildcards=none"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + // Users with full privileges get an empty result, like expected due to the expand_wildcards=none option + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // The dnfof implementation has the effect that the expand_wildcards=none option is disregarded + // Additionally, the dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_noPattern_allowNoIndicesFalse() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_search?size=1000&allow_no_indices=false"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(isForbidden()) + ); + } else { + // The dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.reference(READ)).whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void search_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // The dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_all_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000&expand_wildcards=none"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + // Users with full privileges get an empty result, like expected due to the expand_wildcards=none option + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // The dnfof implementation has the effect that the expand_wildcards=none option is disregarded + // Additionally, the dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_all_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_all/_search?size=1000&expand_wildcards=all"); + + assertThat( + httpResponse, + containsExactly( + clusterConfig.systemIndexPrivilegeEnabled || user == SUPER_UNLIMITED_USER + ? ALL_INDICES + : ALL_INDICES_EXCEPT_SYSTEM_INDICES + ).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*/_search?size=1000"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // The dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_wildcard_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*/_search?size=1000&expand_wildcards=none"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + // Users with full privileges get an empty result, like expected due to the expand_wildcards=none option + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + // The dnfof implementation has the effect that the expand_wildcards=none option is disregarded + // Additionally, the dnfof implementation has the effect that hidden indices might be included even though not requested + assertThat( + httpResponse, + containsExactly(clusterConfig.systemIndexPrivilegeEnabled ? ALL_INDICES : ALL_INDICES_EXCEPT_SYSTEM_INDICES).at( + "hits.hits[*]._index" + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_wildcard_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*/_search?size=1000&expand_wildcards=all"); + + assertThat( + httpResponse, + containsExactly( + clusterConfig.systemIndexPrivilegeEnabled || user == SUPER_UNLIMITED_USER + ? ALL_INDICES + : ALL_INDICES_EXCEPT_SYSTEM_INDICES + ).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_staticIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a1/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) + ); + } + } + + @Test + public void search_staticIndices_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a1,index_b1/_search?size=1000&ignore_unavailable=true"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_b1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_staticIndices_nonExisting() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_ax/_search?size=1000"); + + if (containsExactly(index_ax).reducedBy(user.reference(READ)).isEmpty()) { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/search]")); + } else { + assertThat(httpResponse, isNotFound()); + } + } + } + + @Test + public void search_staticIndices_hidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_hidden/_search?size=1000"); + assertThat( + httpResponse, + containsExactly(index_hidden).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.reference(READ)) + ); + } + } + + @Test + public void search_staticIndices_systemIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get(".system_index_plugin/_search?size=1000"); + if (clusterConfig.systemIndexPrivilegeEnabled || user == SUPER_UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(system_index_plugin).at("hits.hits[*]._index").butForbiddenIfIncomplete(user.reference(READ)) + ); + } else { + // legacy privilege evaluation without system index privilege enabled + if (user == UNLIMITED_USER || user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { + // The legacy evaluation grants access in SystemIndexAccessPrivilegesEvaluator for users with * privileges, + // but withholds documents on the DLS level + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/search]")); + } + } + } + } + + @Test + public void search_staticIndices_systemIndex_alias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get(".alias_with_system_index/_search?size=1000"); + + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly(system_index_plugin).at("hits.hits[*]._index")); + } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION) { + if (user == UNLIMITED_USER || user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { + // The legacy evaluation grants access in SystemIndexAccessPrivilegesEvaluator for users with * privileges, + // but withholds documents on the DLS level + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index")); + } else { + assertThat(httpResponse, isForbidden()); + } + } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { + assertThat( + httpResponse, + containsExactly(system_index_plugin).at("hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) + ); + } else { + if (user.reference(READ).covers(alias_with_system_index)) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly(system_index_plugin).at("hits.hits[*]._index")); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } + } + + @Test + public void search_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a*,index_b*/_search?size=1000"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_indexPattern_minus() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a*,index_b*,-index_b2,-index_b3/_search?size=1000"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_indexPattern_nonExistingIndex_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get( + "index_a*,index_b*,xxx_non_existing/_search?size=1000&ignore_unavailable=true" + ); + + // The presence of a non existing index has the effect that the other patterns are not resolved by IndexResolverReplacer + // This causes a few more 403 errors where the granted index patterns do not use wildcards + + if (user == LIMITED_USER_B1 || user == LIMITED_USER_ALIAS_AB1) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void search_indexPattern_noWildcards() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a*,index_b*/_search?size=1000&expand_wildcards=none"); + // We have to specify the users here explicitly because here we need to check privileges for the + // non-existing (and invalidly named) indices "index_a*" and "index_b*". + // However: Again, dnfof gets the indices options wrong and ignores the expand_wildcards=none flag when getting active + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + // Only these users "get through". Because the indices does not exist, they get a 404 + assertThat(httpResponse, isNotFound()); + } else { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + + } + } + + @Test + public void search_indexPatternAndStatic_negation() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + // If there is a wildcard, negation will also affect indices specified without a wildcard + TestRestClient.HttpResponse httpResponse = restClient.get("index_a*,index_b1,index_b2,-index_b2/_search?size=1000"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_indexPattern_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*index*/_search?size=1000&expand_wildcards=all"); + + if (user == SUPER_UNLIMITED_USER) { + // The super admin sees everything + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + ).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else if (!clusterConfig.systemIndexPrivilegeEnabled) { + // Without system index privileges, the system_index_plugin will be never included + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1, index_hidden, index_hidden_dot) + .at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // Things get buggy here; basically all requests fail with a 403 + if (user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { + // This user is supposed to have the system index privilege for the index .system_index_plugin + // However, the system index privilege evaluation code only works correct when the system index is the + // only requested index. If also non system indices are requested in the same request, it will require + // the presence of the system index privilege for all indices. As this is not the case, the request + // will be denied with a 403 error. + assertThat(httpResponse, isForbidden()); + } else { + // The other users do not have privileges for the system index. The dnfof feature promises to filter + // out indices without authorization from eligible requests. However, the SystemIndexAccessEvaluator + // is not aware of this and just denies all these requests + // See also https://github.com/opensearch-project/security/issues/5546 + assertThat(httpResponse, isForbidden()); + } + } + } + } + + @Test + public void search_alias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_ab1/_search?size=1000"); + if (clusterConfig.legacyPrivilegeEvaluation) { + // The legacy privilege evaluation with dnfof enabled can replace aliases by a sub-set of its member indices + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void search_alias_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_ab1*/_search?size=1000"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_alias_pattern_negation() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_*,-alias_ab1/_search?size=1000"); + // Another interesting effect: The negation on alias names does actually have no effect. + // This is this time a bug in core. TODO: File issue + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void search_alias_pattern_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*alias*/_search?size=1000&expand_wildcards=all"); + + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1, system_index_plugin).at("hits.hits[*]._index") + ); + } else if (user != LIMITED_USER_NONE) { + if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION_SYSTEM_INDEX_PERMISSION) { + // For all users without the system index permission, SystemIndexAccessEvaluator shuts the door + // For the user with the system index permission, that happens as well, as SystemIndexAccessEvaluator expects the + // permission for all requested indices, even if they are not system indices + assertThat(httpResponse, isForbidden()); + } else { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_c1, system_index_plugin).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(isOk()) + ); + } + } else { + assertThat(httpResponse, isForbidden()); + } + } + } + + @Test + public void search_aliasAndIndex_ignoreUnavailable() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_ab1,index_b1/_search?size=1000&ignore_unavailable=true"); + if (clusterConfig.legacyPrivilegeEvaluation) { + // The legacy privilege evaluation with dnfof enabled can replace aliases by a sub-set of its member indices + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("hits.hits[*]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(isForbidden()) + ); + + } + } + } + + @Test + public void search_nonExisting_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist/_search?size=1000"); + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isNotFound()); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/search]")); + } + } + } + + @Test + public void search_nonExisting_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist*/_search?size=1000"); + + assertThat(httpResponse, containsExactly().at("hits.hits[*]._index").whenEmpty(isOk())); + } + } + + @Test + public void search_termsAggregation_index() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("/_search", """ + { + "size": 0, + "aggs": { + "indices": { + "terms": { + "field": "_index", + "size": 1000 + } + } + } + }"""); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at( + "aggregations.indices.buckets[*].key" + ).reducedBy(user.reference(READ)).whenEmpty(isOk()) + ); + } + } + + @Test + public void search_pit() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.post("index_a*,index_b*/_search/point_in_time?keep_alive=1m"); + + RestIndexMatchers.OnResponseIndexMatcher indexMatcher = containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3 + ); + + if (indexMatcher.reducedBy(user.reference(READ)).isEmpty()) { + assertThat( + httpResponse, + isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") + ); + } else { + assertThat(httpResponse, isOk()); + String pitId = httpResponse.getTextFromJsonBody("/pit_id"); + httpResponse = restClient.postJson("/_search?size=1000", String.format(""" + { + "pit": { + "id": "%s" + } + } + """, pitId)); + assertThat(httpResponse, isOk()); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.reference(READ))); + } + } + } + + @Test + public void search_pit_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.post("_all/_search/point_in_time?keep_alive=1m"); + + RestIndexMatchers.OnResponseIndexMatcher indexMatcher; + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + indexMatcher = containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1); + } else { + indexMatcher = containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot + ); + } + + if (indexMatcher.reducedBy(user.reference(READ)).isEmpty()) { + assertThat( + httpResponse, + isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") + ); + } else { + assertThat(httpResponse, isOk()); + String pitId = httpResponse.getTextFromJsonBody("/pit_id"); + httpResponse = restClient.postJson("/_search?size=1000", String.format(""" + { + "pit": { + "id": "%s" + } + } + """, pitId)); + if (clusterConfig.systemIndexPrivilegeEnabled && user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { + // The current request mixes access to a normal index and a system index. + // The current system index permission implementation has the issue that it also + // expects the system index permission for the normal issue then. + // As this is not present, the request https://github.com/opensearch-project/security/issues/5508 + assertThat(httpResponse, isForbidden()); + } else { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.reference(READ))); + } + } + } + } + + @Test + public void search_pit_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.post("index_a1/_search/point_in_time?keep_alive=1m"); + + RestIndexMatchers.OnResponseIndexMatcher indexMatcher = containsExactly(index_a1); + + if (indexMatcher.reducedBy(user.reference(READ)).isEmpty()) { + assertThat( + httpResponse, + isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") + ); + } else { + assertThat(httpResponse, isOk()); + String pitId = httpResponse.getTextFromJsonBody("/pit_id"); + httpResponse = restClient.postJson("/_search?size=1000", String.format(""" + { + "pit": { + "id": "%s" + } + } + """, pitId)); + assertThat(httpResponse, indexMatcher.at("hits.hits[*]._index").reducedBy(user.reference(READ))); + } + } + } + + @Test + public void search_pit_wrongIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.post("index_a*/_search/point_in_time?keep_alive=1m"); + + if (user.reference(READ).coversAll(index_a1, index_a2, index_a3)) { + assertThat(httpResponse, isOk()); + String pitId = httpResponse.getTextFromJsonBody("/pit_id"); + httpResponse = restClient.postJson("index_b*/_search?size=1000", String.format(""" + { + "pit": { + "id": "%s" + } + } + """, pitId)); + assertThat(httpResponse, isBadRequest("/error/root_cause/0/reason", "[indices] cannot be used with point in time")); + + } else { + assertThat( + httpResponse, + isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/point_in_time/create]") + ); + } + } + } + + /** + * Moved from https://github.com/opensearch-project/security/blob/eb7153d772e9e00d49d9cb5ffafb33b5f02399fc/src/integrationTest/java/org/opensearch/security/privileges/PrivilegesEvaluatorTest.java#L103 + * See also https://github.com/opensearch-project/security/issues/1678 + */ + @Test + public void search_template_staticIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + String params = """ + { + "department": [%s] + }""".formatted(TestData.DEPARTMENTS.stream().map(s -> '"' + s + '"').collect(joining(","))); + String query = """ + { + "query": { + "terms": { + "attr_text_1": [ + "{{#department}}", + "{{.}}", + "{{/department}}" + ] + } + } + } + """; + + TestRestClient.HttpResponse httpResponse = restClient.getWithJsonBody("index_a1/_search/template?size=1000", """ + { + "params": %s, + "source": "%s" + }""".formatted(params, escapeJson(query))); + + assertThat( + httpResponse, + containsExactly(index_a1).at("hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) + ); + } + } + + @Test + public void msearch_staticIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("/_msearch", """ + {"index":"index_b1"} + {"size":10, "query":{"bool":{"must":{"match_all":{}}}}} + {"index":"index_b2"} + {"size":10, "query":{"bool":{"must":{"match_all":{}}}}} + """); + assertThat( + httpResponse, + containsExactly(index_b1, index_b2).at("responses[*].hits.hits[*]._index").reducedBy(user.reference(READ)).whenEmpty(isOk()) + ); + } + } + + @Test + public void mget() throws Exception { + TestData.TestDocument testDocumentA1 = index_a1.anyDocument(); + TestData.TestDocument testDocumentB1 = index_b1.anyDocument(); + TestData.TestDocument testDocumentB2 = index_b2.anyDocument(); + + String mget = String.format(""" + { + "docs": [ + { "_index": "index_a1", "_id": "%s" }, + { "_index": "index_b1", "_id": "%s" }, + { "_index": "index_b2", "_id": "%s" } + ] + } + """, testDocumentA1.id(), testDocumentB1.id(), testDocumentB2.id()); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("/_mget", mget); + assertThat( + httpResponse, + containsExactly(index_a1, index_b1, index_b2).at("docs[?(@.found == true)]._index") + .reducedBy(user.reference(READ)) + .whenEmpty(isOk()) + ); + } + } + + @Test + public void mget_alias() throws Exception { + TestData.TestDocument testDocumentC1a = index_c1.anyDocument(); + TestData.TestDocument testDocumentC1b = index_c1.anyDocument(); + + String mget = String.format(""" + { + "docs": [ + { "_index": "alias_c1", "_id": "%s" }, + { "_index": "alias_c1", "_id": "%s" } + ] + } + """, testDocumentC1a.id(), testDocumentC1b.id()); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("/_mget", mget); + assertThat( + httpResponse, + containsExactly(index_c1).at("docs[?(@.found == true)]._index").reducedBy(user.reference(READ)).whenEmpty(isOk()) + ); + } + } + + @Test + public void get() throws Exception { + TestData.TestDocument testDocumentB1 = index_b1.anyDocument(); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_b1/_doc/" + testDocumentB1.id()); + assertThat(httpResponse, containsExactly(index_b1).at("_index").reducedBy(user.reference(READ)).whenEmpty(isForbidden())); + } + } + + @Test + public void get_alias() throws Exception { + TestData.TestDocument testDocumentC1 = index_c1.anyDocument(); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_c1/_doc/" + testDocumentC1.id()); + assertThat(httpResponse, containsExactly(index_c1).at("_index").reducedBy(user.reference(READ)).whenEmpty(isForbidden())); + } + } + + @Test + public void get_systemIndex() throws Exception { + TestData.TestDocument testDocument = system_index_plugin.anyDocument(); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get(".system_index_plugin/_doc/" + testDocument.id()); + + if (clusterConfig == ClusterConfig.LEGACY_PRIVILEGES_EVALUATION) { + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly(system_index_plugin).at("_index")); + } else if (user == LIMITED_USER_C_WITH_SYSTEM_INDICES || user == UNLIMITED_USER) { + // If the user has a role that grants access to the index, they can + // successfully access the index (i.e., they won't get a 403), but + // the index will appear empty (i.e., they will get a 404) + assertThat(httpResponse, isNotFound()); + } else { + assertThat(httpResponse, isForbidden()); + } + } else if ((clusterConfig.systemIndexPrivilegeEnabled && user == LIMITED_USER_C_WITH_SYSTEM_INDICES) + || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly(system_index_plugin).at("_index")); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } + + @Test + public void cat_indices_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/indices?format=json"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("$[*].index") + .reducedBy(user.reference(READ)) + ); + + } else { + // Also here, dnfof might introduce hidden indices even though they were not requested + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden_dot, + index_hidden, + system_index_plugin + ).at("$[*].index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void cat_indices_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/indices/index_a*?format=json"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3).at("$[*].index") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void cat_indices_all_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/indices?format=json&expand_wildcards=all"); + if (user == UNLIMITED_USER) { + assertThat(httpResponse, containsExactly(ALL_INDICES).at("$[*].index")); + } else { + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("$[*].index").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void cat_aliases_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/aliases?format=json"); + + if (clusterConfig.legacyPrivilegeEvaluation && user == UNLIMITED_USER) { + assertThat(httpResponse, containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$[*].alias")); + } else { + if (!user.reference(GET_ALIAS).isEmpty()) { + assertThat( + httpResponse, + containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$[*].alias") + .reducedBy(user.reference(GET_ALIAS)) + .whenEmpty(isOk()) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } + } + + @Test + public void cat_aliases_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/aliases/alias_a*?format=json"); + + if (!user.reference(GET_ALIAS).isEmpty()) { + assertThat( + httpResponse, + containsExactly(alias_ab1).at("$[*].alias").reducedBy(user.reference(GET_ALIAS)).whenEmpty(isOk()) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } + + @Test + public void index_stats_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("/_stats"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("indices.keys()") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + // Also here, dnfof can introduce hidden indices even though they were not requested + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + ).at("indices.keys()") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void index_stats_pattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_b*/_stats"); + + assertThat( + httpResponse, + containsExactly(index_b1, index_b2, index_b3).at("indices.keys()") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + + @Test + public void getAlias_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_alias"); + if (user == UNLIMITED_USER) { + // The legacy privilege evaluation also allows regular users access to metadata of the security index + // This is not a security issue, as the metadata are not really security relevant + assertThat(httpResponse, containsExactly(ALL_INDICES).at("$.keys()")); + } else { + assertThat( + httpResponse, + containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$.*.aliases.keys()") + .reducedBy(user.reference(GET_ALIAS)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("$.keys()") + .reducedBy(user.reference(GET_ALIAS)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } + } + + @Test + public void getAlias_staticAlias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_alias/alias_c1"); + if (user == LIMITED_USER_ALIAS_AB1) { + if (clusterConfig.legacyPrivilegeEvaluation) { + // RestGetAliasesAction does some further post processing on the results, thus we get 404 errors in case a non wildcard + // alias was removed + assertThat(httpResponse, isNotFound()); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:admin/aliases/get]")); + } + } else { + assertThat( + httpResponse, + containsExactly(alias_c1).at("$.*.aliases.keys()").reducedBy(user.reference(GET_ALIAS)).whenEmpty(isForbidden()) + ); + assertThat( + httpResponse, + containsExactly(index_c1).at("$.keys()").reducedBy(user.reference(GET_ALIAS)).whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void getAlias_aliasPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_alias/alias_ab*"); + + if (user == LIMITED_USER_ALIAS_AB1 || user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly(alias_ab1).at("$.*.aliases.keys()").reducedBy(user.reference(GET_ALIAS))); + assertThat(httpResponse, containsExactly(index_a1, index_a2, index_a3, index_b1).at("$.keys()")); + } else if (user == LIMITED_USER_ALIAS_C1 || user == LIMITED_USER_C_WITH_SYSTEM_INDICES) { + // This is also a kind of anomaly in the legacy privilege evaluation: Even though we do not have permissions + // we get a 200 response with an empty result + assertThat(httpResponse, isOk()); + assertTrue(httpResponse.getBody(), httpResponse.bodyAsMap().isEmpty()); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:admin/aliases/get]")); + } + } + } + + @Test + public void getAlias_indexPattern_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("*index*/_alias?expand_wildcards=all"); + if (user == SUPER_UNLIMITED_USER) { + // The super admin sees everything + assertThat(httpResponse, isOk()); + assertThat(httpResponse, containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$.*.aliases.keys()")); + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + ).at("$.keys()") + ); + } else if (!clusterConfig.systemIndexPrivilegeEnabled) { + if (user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + ).at("$.keys()") + ); + } else { + assertThat( + httpResponse, + containsExactly(alias_ab1, alias_c1, alias_with_system_index).at("$.*.aliases.keys()") + .reducedBy(user.reference(GET_ALIAS)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("$.keys()") + .reducedBy(user.reference(GET_ALIAS)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + } else { + // If the system index privilege is enabled, we only get 403 errors, as SystemIndexPrivilegeEvaluator + // is not aware of dnfof; see https://github.com/opensearch-project/security/issues/5546 + assertThat(httpResponse, isForbidden()); + } + } + } + + @Test + public void analyze_noIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("_analyze", "{\"text\": \"sample text\"}"); + + // _analyze without index is different from most other operations: + // Usually, the absence of an index means "all indices". For analyze, however, it means: "no index". + // However, the IndexResolverReplacer does not get this right; it assumes that all indices are requested. + // Thus, we get only through to this operation with full privileges for all indices + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:admin/analyze]")); + } + } + } + + @Test + public void analyze_staticIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.postJson("index_a1/_analyze", "{\"text\": \"sample text\"}"); + IndexMatcher matcher = containsExactly(index_a1).reducedBy(user.reference(READ)); + + if (matcher.isEmpty()) { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:admin/analyze]")); + } else { + assertThat(httpResponse, isOk()); + } + } + } + + @Test + public void resolve_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/*"); + + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1, alias_ab1, alias_c1).at( + "$.*[*].name" + ).reducedBy(user.reference(READ)).whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } else { + assertThat( + httpResponse, + containsExactly( + index_a1, + index_a2, + index_a3, + index_b1, + index_b2, + index_b3, + index_c1, + index_hidden, + index_hidden_dot, + system_index_plugin + ).at("$.*[*].name") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + } + + } + } + + @Test + public void resolve_wildcard_includeHidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/*?expand_wildcards=all"); + + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + // The legacy privilege evaluation also allows regular users access to metadata of the security index + // This is not a security issue, as the metadata are not really security relevant + assertThat(httpResponse, containsExactly(ALL_INDICES_AND_ALIASES).at("$.*[*].name")); + } else { + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("$.*[*].name").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void resolve_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_resolve/index/index_a*,index_b*"); + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3).at("$.*[*].name") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + + } + } + + @Test + public void field_caps_all() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + if (user == SUPER_UNLIMITED_USER || user == UNLIMITED_USER) { + TestRestClient.HttpResponse httpResponse = restClient.get("_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3, index_c1).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + + } else { + TestRestClient.HttpResponse httpResponse = restClient.get("_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(ALL_INDICES).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + + } + + } + } + + @Test + public void field_caps_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_b*/_field_caps?fields=*"); + + assertThat( + httpResponse, + containsExactly(index_b1, index_b2, index_b3).at("indices").reducedBy(user.reference(READ)).whenEmpty(isForbidden()) + ); + } + } + + @Test + public void field_caps_staticIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a1/_field_caps?fields=*"); + assertThat(httpResponse, containsExactly(index_a1).at("indices").reducedBy(user.reference(READ)).whenEmpty(isForbidden())); + } + } + + @Test + public void field_caps_staticIndices_hidden() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_hidden/_field_caps?fields=*"); + assertThat(httpResponse, containsExactly(index_hidden).at("indices").butForbiddenIfIncomplete(user.reference(READ))); + } + } + + @Test + public void field_caps_alias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_ab1/_field_caps?fields=*"); + if (clusterConfig.legacyPrivilegeEvaluation) { + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(isForbidden()) + ); + } + } + } + + @Test + public void field_caps_aliasPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("alias_ab*/_field_caps?fields=*"); + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + + } + } + + @Test + public void field_caps_nonExisting_static() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_ax/_field_caps?fields=*"); + + if (containsExactly(index_ax).reducedBy(user.reference(READ)).isEmpty()) { + assertThat(httpResponse, isForbidden("/error/root_cause/0/reason", "no permissions for [indices:data/read/field_caps]")); + } else { + assertThat(httpResponse, isNotFound()); + } + } + } + + @Test + public void field_caps_nonExisting_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("x_does_not_exist*/_field_caps?fields=*"); + + // As this resolves to an empty set of indices, we are always allowed + assertThat(httpResponse, containsExactly().at("indices").whenEmpty(isOk())); + } + } + + @Test + public void field_caps_indexPattern_minus() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("index_a*,index_b*,-index_b2,-index_b3/_field_caps?fields=*"); + + assertThat( + httpResponse, + containsExactly(index_a1, index_a2, index_a3, index_b1).at("indices") + .reducedBy(user.reference(READ)) + .whenEmpty(clusterConfig.allowsEmptyResultSets ? isOk() : isForbidden()) + ); + + } + } + + @Test + public void pit_list_all() throws Exception { + String indexA1pitId = createPit(index_a1); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_search/point_in_time/_all"); + + // At the moment, it is sufficient to have any privileges for any existing index to use the _all API + // This is clearly a bug; yet, not a severe issue, as we do not have really sensitive things available here. + // This is caused by the following line which makes PrivilegesEvaluator believe it could reduce the indices + // to authorized indices, even though it actually could not: + // https://github.com/opensearch-project/security/blob/aee54a8ca2a6cc596cb1e490be1e9fa240286246/src/main/java/org/opensearch/security/resolver/IndexResolverReplacer.java#L824-L825 + if (user != LIMITED_USER_NONE && user != LIMITED_USER_OTHER_PRIVILEGES) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + deletePit(indexA1pitId); + } + } + + @Test + public void pit_delete() throws Exception { + String indexA1pitId = createPit(index_a1); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.delete("_search/point_in_time", json("pit_id", List.of(indexA1pitId))); + + if (user.reference(READ).covers(index_a1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + deletePit(indexA1pitId); + } + } + + @Test + public void pit_catSegments() throws Exception { + String indexA1pitId = createPit(index_a1); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/pit_segments", json("pit_id", List.of(indexA1pitId))); + + if (user.reference(READ).covers(index_a1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + deletePit(indexA1pitId); + } + } + + @Test + public void pit_catSegments_all() throws Exception { + String indexA1pitId = createPit(index_a1); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.get("_cat/pit_segments/_all"); + + // The user needs to have the privilege for all indices. If it is only granted for a subset of indices, this will be forbidden. + if (user == UNLIMITED_USER || user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + deletePit(indexA1pitId); + } + } + + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") + public static Collection params() { + List result = new ArrayList<>(); + + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + for (TestSecurityConfig.User user : USERS) { + result.add(new Object[] { clusterConfig, user, user.getDescription() }); + } + } + return result; + } + + public IndexAuthorizationReadOnlyIntTests(ClusterConfig clusterConfig, TestSecurityConfig.User user, String description) + throws Exception { + this.user = user; + this.cluster = clusterConfig.cluster(IndexAuthorizationReadOnlyIntTests::clusterBuilder); + this.clusterConfig = clusterConfig; + } + + public static class SystemIndexTestPlugin extends Plugin implements SystemIndexPlugin { + @Override + public Collection getSystemIndexDescriptors(Settings settings) { + return List.of( + new SystemIndexDescriptor(".system_index_plugin", "for testing system index exclusion"), + new SystemIndexDescriptor(".system_index_plugin_not_existing", "for testing system index exclusion") + ); + } + } + + private String createPit(TestIndex... indices) throws IOException { + try (TestRestClient client = cluster.getAdminCertRestClient()) { + TestRestClient.HttpResponse response = client.post( + Stream.of(indices).map(TestIndex::name).collect(joining(",")) + "/_search/point_in_time?keep_alive=1m" + ); + assertThat(response, isOk()); + return response.getTextFromJsonBody("/pit_id"); + } + } + + private void deletePit(String... pitIds) { + try (TestRestClient client = cluster.getAdminCertRestClient()) { + TestRestClient.HttpResponse response = client.delete("_search/point_in_time", json("pit_id", Arrays.asList(pitIds))); + assertThat(response, isOk()); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java new file mode 100644 index 0000000000..fe685bc2a0 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/IndexAuthorizationReadWriteIntTests.java @@ -0,0 +1,1183 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.security.privileges.int_tests; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import javax.annotation.concurrent.NotThreadSafe; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.google.common.collect.ImmutableList; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.action.admin.indices.open.OpenIndexRequest; +import org.opensearch.action.admin.indices.refresh.RefreshRequest; +import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.TestSecurityConfig.Role; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import org.opensearch.test.framework.data.TestAlias; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.matcher.RestIndexMatchers; +import org.opensearch.transport.client.Client; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.cluster.TestRestClient.json; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimited; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestMatchers.isBadRequest; +import static org.opensearch.test.framework.matcher.RestMatchers.isCreated; +import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; +import static org.opensearch.test.framework.matcher.RestMatchers.isNotFound; +import static org.opensearch.test.framework.matcher.RestMatchers.isOk; + +/** + * This class defines a huge test matrix for index related access controls. This class is especially for read/write operations on indices and aliases. + * It uses the following dimensions: + *

    + *
  • ClusterConfig: At the moment, we test without and with system index permission enabled. New semantics will follow later.
    • + *
  • TestSecurityConfig.User: We have quite a few of different users with different privileges configurations.
    • + *
  • The test methods represent different operations with different options that are tested
    • + *
+ * To cope with the huge space of tests, this class uses test oracles to verify the result of the operations. + * These are defined with the "indexMatcher()" method of TestSecurityConfig.User. See there and the class IndexApiMatchers. + */ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +@NotThreadSafe +public class IndexAuthorizationReadWriteIntTests { + + // ------------------------------------------------------------------------------------------------------- + // Test indices used by this test suite. We use the following naming scheme: + // - index_*r*: This test will not write to this index + // - index_*w*: This test can write to this index; the test won't delete and recreate it + // - index_*wx*: The index is not initially created; the test can create it on demand and delete it again + // ------------------------------------------------------------------------------------------------------- + + static final TestIndex index_ar1 = TestIndex.name("index_ar1").documentCount(10).build(); + static final TestIndex index_ar2 = TestIndex.name("index_ar2").documentCount(10).build(); + static final TestIndex index_aw1 = TestIndex.name("index_aw1").documentCount(10).build(); + static final TestIndex index_aw2 = TestIndex.name("index_aw2").documentCount(10).build(); + static final TestIndex index_br1 = TestIndex.name("index_br1").documentCount(10).build(); + static final TestIndex index_br2 = TestIndex.name("index_br2").documentCount(10).build(); + static final TestIndex index_bw1 = TestIndex.name("index_bw1").documentCount(10).build(); + static final TestIndex index_bw2 = TestIndex.name("index_bw2").documentCount(10).build(); + static final TestIndex index_cr1 = TestIndex.name("index_cr1").documentCount(10).build(); + static final TestIndex index_cw1 = TestIndex.name("index_cw1").documentCount(10).build(); + static final TestIndex index_hidden = TestIndex.name("index_hidden").hidden().documentCount(1).seed(8).build(); + static final TestIndex system_index_plugin = TestIndex.name(".system_index_plugin").hidden().documentCount(1).seed(8).build(); + static final TestIndex system_index_plugin_not_existing = TestIndex.name(".system_index_plugin_not_existing") + .hidden() + .documentCount(0) + .build(); // not initially created + + static final TestAlias alias_ab1r = new TestAlias("alias_ab1r").on(index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_bw1); + static final TestAlias alias_ab1w = new TestAlias("alias_ab1w").on(index_aw1, index_aw2, index_bw1).writeIndex(index_aw1); + static final TestAlias alias_ab1w_nowriteindex = new TestAlias("alias_ab1w_nowriteindex").on(index_aw1, index_aw2, index_bw1); + + static final TestAlias alias_c1 = new TestAlias("alias_c1", index_cr1, index_cw1); + + static final TestIndex index_bwx1 = TestIndex.name("index_bwx1").documentCount(0).build(); // not initially created + static final TestIndex index_bwx2 = TestIndex.name("index_bwx2").documentCount(0).build(); // not initially created + + static final TestAlias alias_bwx = new TestAlias("alias_bwx"); // not initially created + + static final List ALL_NON_HIDDEN_INDICES = List.of( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_cr1, + index_cw1 + ); + + static final List ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES = List.of( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_cr1, + index_cw1, + alias_ab1w, + alias_ab1r, + alias_c1, + alias_ab1w_nowriteindex, + index_hidden + ); + + /** + * This key identifies assertion reference data for index search/read permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey READ = new TestSecurityConfig.User.MetadataKey<>( + "read", + RestIndexMatchers.IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for index write permissions of individual users. This does + * not include index creation permissions. + */ + static final TestSecurityConfig.User.MetadataKey WRITE = new TestSecurityConfig.User.MetadataKey<>( + "write", + RestIndexMatchers.IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for create index permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey CREATE_INDEX = + new TestSecurityConfig.User.MetadataKey<>("create_index", RestIndexMatchers.IndexMatcher.class); + + /** + * This key identifies assertion reference data for manage index permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey MANAGE_INDEX = + new TestSecurityConfig.User.MetadataKey<>("manage_index", RestIndexMatchers.IndexMatcher.class); + + /** + * This key identifies assertion reference data for alias management permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey MANAGE_ALIAS = + new TestSecurityConfig.User.MetadataKey<>("manage_alias", RestIndexMatchers.IndexMatcher.class); + + // ------------------------------------------------------------------------------------------------------- + // Test users with which the tests will be executed; the users need to be added to the list USERS below + // The users have two redundant versions or privilege configuration, which needs to be kept in sync: + // - The standard role configuration defined with .roles() + // - IndexMatchers which act as test oracles, defined with the indexMatcher() methods + // ------------------------------------------------------------------------------------------------------- + + /** + * A simple user that can read from index_a* and write to index_aw*; the user as no privileges to create or manage indices + */ + static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// + .description("index_a*")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("index_a*")// + .indexPermissions("write") + .on("index_aw*") + )// + .reference(READ, limitedTo(index_ar1, index_ar2, index_aw1, index_aw2))// + .reference(WRITE, limitedTo(index_aw1, index_aw2))// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); + + /** + * A simple user that can read from index_b* and write to index_bw*; the user as no privileges to create or manage indices + */ + static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// + .description("index_b*")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("index_b*")// + .indexPermissions("write") + .on("index_bw*") + )// + .reference(READ, limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); + + /** + * A simple user that can read from index_b* and write to index_bw*; additionally, they can create index_bw* indices + */ + static TestSecurityConfig.User LIMITED_USER_B_CREATE_INDEX = new TestSecurityConfig.User("limited_user_B_create_index")// + .description("index_b* with create index privs")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("index_b*")// + .indexPermissions("write") + .on("index_bw*")// + .indexPermissions("create_index") + .on("index_bw*") + )// + .reference(READ, limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(CREATE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); + + /** + * A simple user that can read from index_b* and write to index_bw*; additionally, they can create and manage index_bw* indices + */ + static TestSecurityConfig.User LIMITED_USER_B_MANAGE_INDEX = new TestSecurityConfig.User("limited_user_B_manage_index")// + .description("index_b* with manage privs")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("index_b*")// + .indexPermissions("write") + .on("index_bw*")// + .indexPermissions("manage") + .on("index_bw*") + )// + .reference(READ, limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(CREATE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(MANAGE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(MANAGE_ALIAS, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2)); + + /** + * A user that can read from index_b* and write to index_bw*; they can create and manage index_bw* indices and manage alias_bwx* aliases. + * For users with such alias permissions, keep in mind that alias permissions are inherited by the member indices. + * Thus, indices can gain or lose privileges when they are added/removed from the alias. + */ + static TestSecurityConfig.User LIMITED_USER_B_MANAGE_INDEX_ALIAS = new TestSecurityConfig.User("limited_user_B_manage_index_alias")// + .description("index_b*, alias_bwx* with manage privs")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("index_b*")// + .indexPermissions("write") + .on("index_bw*")// + .indexPermissions("manage") + .on("index_bw*")// + .indexPermissions("crud", "manage", "manage_aliases") + .on("alias_bwx*") + )// + .reference(READ, limitedTo(index_br1, index_br2, index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(CREATE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2))// + .reference(MANAGE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx))// + .reference(MANAGE_ALIAS, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx)); + + /** + * This user differs from LIMITED_USER_B_MANAGE_INDEX_ALIAS the way that it does not give any direct + * write privileges to index_bw*; rather, it gives write privileges to alias_bxw. Any index which happens + * to be member of that alias then gains these write privileges. + */ + static TestSecurityConfig.User LIMITED_USER_B_READ_ONLY_MANAGE_INDEX_ALIAS = new TestSecurityConfig.User( + "limited_user_B_index_read_only_manage_index_alias" + )// + .description("index_b* r/o, alias_bwx* r/w with manage privs")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("index_b*")// + .indexPermissions("crud", "manage", "manage_aliases") + .on("alias_bwx*") + )// + .reference(READ, limitedTo(index_br1, index_br2))// + .reference(WRITE, limitedToNone())// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedTo(alias_bwx))// + .reference(MANAGE_ALIAS, limitedTo(alias_bwx)); + + /** + * Same as LIMITED_USER_B_MANAGE_INDEX_ALIAS with the addition of read/write/manage privileges on index_hidden* + */ + static TestSecurityConfig.User LIMITED_USER_B_HIDDEN_MANAGE_INDEX_ALIAS = new TestSecurityConfig.User( + "limited_user_B_hidden_manage_index_alias" + )// + .description("index_b*, index_hidden*, alias_bwx* with manage privs, index_a* read only")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor") + .on("index_a*", "index_b*", "index_hidden*")// + .indexPermissions("write") + .on("index_bw*", "index_hidden*")// + .indexPermissions("manage") + .on("index_bw*", "index_hidden*")// + .indexPermissions("crud", "manage", "manage_aliases") + .on("alias_bwx*") + )// + .reference( + READ, + limitedTo( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_bwx1, + index_bwx2, + index_hidden + ) + )// + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// + .reference(CREATE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, index_hidden))// + .reference(MANAGE_INDEX, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx, index_hidden))// + .reference(MANAGE_ALIAS, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, alias_bwx, index_hidden)); + + /** + * Same as LIMITED_USER_B with the addition of read/write/manage privileges for ".system_index_plugin", ".system_index_plugin_*" + * including the explicit "system:admin/system_index" privilege. + */ + static TestSecurityConfig.User LIMITED_USER_B_SYSTEM_INDEX_MANAGE = new TestSecurityConfig.User("limited_user_B_system_index_manage")// + .description("index_b*, .system_index_plugin with manage privs")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "system:admin/system_index") + .on("index_b*", "index_hidden*", ".system_index_plugin")// + .indexPermissions("write", "system:admin/system_index") + .on("index_bw*", ".system_index_plugin", ".system_index_plugin_*")// + .indexPermissions("manage", "system:admin/system_index") + .on("index_bw*", ".system_index_plugin", ".system_index_plugin_*") + )// + .reference( + READ, + limitedTo( + index_br1, + index_br2, + index_bw1, + index_bw2, + index_bwx1, + index_bwx2, + system_index_plugin, + system_index_plugin_not_existing + ) + )// + .reference(WRITE, limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing))// + .reference( + CREATE_INDEX, + limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing) + )// + .reference( + MANAGE_INDEX, + limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing) + )// + .reference( + MANAGE_ALIAS, + limitedTo(index_bw1, index_bw2, index_bwx1, index_bwx2, system_index_plugin, system_index_plugin_not_existing) + ); + + /** + * A simple test user that has read privileges on alias_ab1r and write privileges on alias_ab1w*. The user + * has no direct privileges on indices; all privileges are gained via the aliases. + */ + static TestSecurityConfig.User LIMITED_USER_AB1_ALIAS = new TestSecurityConfig.User("limited_user_alias_AB1")// + .description("alias_ab1")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "indices_monitor", "indices:admin/aliases/get") + .on("alias_ab1r")// + .indexPermissions("read", "indices_monitor", "indices:admin/aliases/get", "write") + .on("alias_ab1w*") + )// + .reference( + READ, + limitedTo(index_ar1, index_ar2, index_aw1, index_aw2, index_br1, index_bw1, alias_ab1r, alias_ab1w, alias_ab1w_nowriteindex) + )// + .reference(WRITE, limitedTo(index_aw1, index_aw2, index_bw1, alias_ab1w, alias_ab1w_nowriteindex))// + .reference(CREATE_INDEX, limitedTo(index_aw1, index_aw2, index_bw1))// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); + /** + * A simple test user that has read/only privileges on alias_ab1r and alias_ab1w*. However, they have write + * privileges for the member index index_aw1. + */ + static TestSecurityConfig.User LIMITED_USER_AB1_ALIAS_READ_ONLY = new TestSecurityConfig.User("limited_user_alias_AB1_read_only")// + .description("read/only on alias_ab1w, but with write privs in write index index_aw1")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read", "write", "indices:admin/refresh") + .on("index_aw1")// + .indexPermissions("read") + .on("alias_ab1w") + )// + .reference(READ, limitedTo(index_aw1, index_aw2, index_bw1, alias_ab1w))// + .reference(WRITE, limitedTo(index_aw1))// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); + + /** + * A simple test user which has read/only privileges for "*" + */ + static TestSecurityConfig.User LIMITED_READ_ONLY_ALL = new TestSecurityConfig.User("limited_read_only_all")// + .description("read/only on *")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read") + .on("*") + )// + .reference(READ, unlimited())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); + + /** + * A simple test user which has read/only privileges for "index_a*" + */ + static TestSecurityConfig.User LIMITED_READ_ONLY_A = new TestSecurityConfig.User("limited_read_only_A")// + .description("read/only on index_a*")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("read") + .on("index_a*") + )// + .reference(READ, limitedTo(index_ar1, index_ar2, index_aw1, index_aw2))// + .reference(WRITE, limitedToNone())// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); + + /** + * A simple test user that only has index privileges for indices that are not used by this test. + */ + static TestSecurityConfig.User LIMITED_USER_OTHER_PRIVILEGES = new TestSecurityConfig.User("limited_user_other_privileges")// + .description("no privileges for existing indices")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("crud", "indices_monitor") + .on("index_does_not_exist_*") + )// + .reference(READ, limitedToNone())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); + + /** + * A simple test user that has no index privileges at all. + */ + static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// + .description("no index privileges")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + )// + .reference(READ, limitedToNone())// + .reference(WRITE, limitedToNone())// + .reference(CREATE_INDEX, limitedToNone())// + .reference(MANAGE_INDEX, limitedToNone())// + .reference(MANAGE_ALIAS, limitedToNone()); + + /** + * A user with "*" privileges on "*"; as it is a regular user, they are still subject to system index + * restrictions and similar things. + */ + static TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// + .description("unlimited")// + .roles( + new Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor")// + .indexPermissions("*") + .on("*")// + .indexPermissions("*") + .on("*") + )// + .reference(READ, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .reference(WRITE, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .reference(CREATE_INDEX, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .reference(MANAGE_INDEX, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx))// + .reference(MANAGE_ALIAS, limitedTo(ALL_INDICES_AND_ALIASES_EXCEPT_SYSTEM_INDICES).and(index_bwx1, index_bwx2, alias_bwx)); + + /** + * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. + * This serves as a base for comparison with the default behavior. + */ + static TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// + .description("super unlimited (admin cert)")// + .adminCertUser()// + .reference(READ, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(WRITE, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(CREATE_INDEX, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(MANAGE_INDEX, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(MANAGE_ALIAS, unlimitedIncludingOpenSearchSecurityIndex()); + + static List USERS = ImmutableList.of( + LIMITED_USER_A, + LIMITED_USER_B, + LIMITED_USER_B_CREATE_INDEX, + LIMITED_USER_B_MANAGE_INDEX, + LIMITED_USER_B_MANAGE_INDEX_ALIAS, + LIMITED_USER_B_READ_ONLY_MANAGE_INDEX_ALIAS, + LIMITED_USER_B_HIDDEN_MANAGE_INDEX_ALIAS, + LIMITED_USER_B_SYSTEM_INDEX_MANAGE, + LIMITED_USER_AB1_ALIAS, + LIMITED_USER_AB1_ALIAS_READ_ONLY, + LIMITED_READ_ONLY_ALL, + LIMITED_READ_ONLY_A, + LIMITED_USER_OTHER_PRIVILEGES, + LIMITED_USER_NONE, + UNLIMITED_USER, + SUPER_UNLIMITED_USER + ); + + static LocalCluster.Builder clusterBuilder() { + return new LocalCluster.Builder().singleNode() + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USERS)// + .indices( + index_ar1, + index_ar2, + index_aw1, + index_aw2, + index_br1, + index_br2, + index_bw1, + index_bw2, + index_cr1, + index_cw1, + index_hidden, + system_index_plugin + )// + .aliases(alias_ab1r, alias_ab1w, alias_ab1w_nowriteindex, alias_c1)// + .nodeSettings(Map.of("action.destructive_requires_name", false)) + .plugin(IndexAuthorizationReadOnlyIntTests.SystemIndexTestPlugin.class); + } + + @AfterClass + public static void stopClusters() { + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + clusterConfig.shutdown(); + } + } + + final TestSecurityConfig.User user; + final LocalCluster cluster; + final ClusterConfig clusterConfig; + + @Test + public void putDocument() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.put("index_bw1/_doc/put_test_1", json("a", 1)); + assertThat(httpResponse, containsExactly(index_bw1).at("_index").reducedBy(user.reference(WRITE)).whenEmpty(isForbidden())); + } finally { + delete("index_bw1/_doc/put_test_1"); + } + } + + @Test + public void putDocument_systemIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + TestRestClient.HttpResponse httpResponse = restClient.put(".system_index_plugin/_doc/put_test_1", json("a", 1)); + if (clusterConfig.systemIndexPrivilegeEnabled && user.reference(WRITE).covers(system_index_plugin)) { + assertThat(httpResponse, isCreated()); + } else if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isCreated()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(".system_index_plugin/_doc/put_test_1"); + } + } + + @Test + public void deleteDocument() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user); TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + + // Initialization + { + HttpResponse httpResponse = adminRestClient.put("index_bw1/_doc/put_delete_test_1?refresh=true", json("a", 1)); + assertThat(httpResponse, isCreated()); + } + + HttpResponse httpResponse = restClient.delete("index_bw1/_doc/put_delete_test_1"); + assertThat(httpResponse, containsExactly(index_bw1).at("_index").reducedBy(user.reference(WRITE)).whenEmpty(isForbidden())); + } finally { + delete("index_bw1/_doc/put_delete_test_1"); + } + } + + @Test + public void deleteByQuery_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user); TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + + HttpResponse httpResponse = adminRestClient.put( + "index_bw1/_doc/put_delete_delete_by_query_b1?refresh=true", + json("delete_by_query_test", "yes") + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "index_bw1/_doc/put_delete_delete_by_query_b2?refresh=true", + json("delete_by_query_test", "no") + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "index_aw1/_doc/put_delete_delete_by_query_a1?refresh=true", + json("delete_by_query_test", "yes") + ); + assertThat(httpResponse, isCreated()); + httpResponse = adminRestClient.put( + "index_aw1/_doc/put_delete_delete_by_query_a2?refresh=true", + json("delete_by_query_test", "no") + ); + assertThat(httpResponse, isCreated()); + + httpResponse = restClient.postJson("index_aw*,index_bw*/_delete_by_query?wait_for_completion=true", """ + { + "query": { + "term": { + "delete_by_query_test": "yes" + } + } + }"""); + + if (clusterConfig.legacyPrivilegeEvaluation) { + // dnfof is not applicable to indices:data/write/delete/byquery, so we need privileges for all indices + if (user.reference(WRITE).coversAll(index_aw1, index_aw2, index_bw1, index_bw2)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + + } finally { + delete( + "index_bw1/_doc/put_delete_delete_by_query_b1", + "index_bw1/_doc/put_delete_delete_by_query_b2", + "index_aw1/_doc/put_delete_delete_by_query_a1", + "index_aw1/_doc/put_delete_delete_by_query_a2" + ); + } + } + + @Test + public void putDocument_bulk() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + RestIndexMatchers.IndexMatcher writePrivileges = user.reference(WRITE); + + HttpResponse httpResponse = restClient.putJson("_bulk", """ + {"index": {"_index": "index_aw1", "_id": "new_doc_aw1"}} + {"a": 1} + {"index": {"_index": "index_bw1", "_id": "new_doc_bw1"}} + {"a": 1} + {"index": {"_index": "index_cw1", "_id": "new_doc_cw1"}} + {"a": 1} + """); + if (user != LIMITED_USER_NONE) { + assertThat( + httpResponse, + containsExactly(index_aw1, index_bw1, index_cw1).at("items[*].index[?(@.result == 'created')]._index") + .reducedBy(writePrivileges) + .whenEmpty(isOk()) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete("index_aw1/_doc/new_doc_aw1", "index_bw1/_doc/new_doc_bw1", "index_cw1/_doc/new_doc_cw1"); + } + } + + @Test + public void putDocument_alias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.put("alias_ab1w/_doc/put_doc_alias_test_1", json("a, 1")); + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.reference(WRITE).coversAll(index_aw1, index_aw2, index_bw1)) { + assertThat(httpResponse, isCreated()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + delete("alias_ab1w/_doc/put_doc_alias_test_1"); + } + } + + @Test + public void putDocument_alias_noWriteIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.put("alias_ab1w_nowriteindex/_doc/put_doc_alias_test_1", json("a, 1")); + + if (containsExactly(alias_ab1w_nowriteindex).reducedBy(user.reference(WRITE)).isEmpty()) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat(httpResponse, isBadRequest()); + } + } + } + + @Test + public void putDocument_bulk_alias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson("_bulk", """ + {"index": {"_index": "alias_ab1w", "_id": "put_doc_alias_bulk_test_1"}} + {"a": 1} + """); + + if (user == LIMITED_USER_A || user == LIMITED_USER_AB1_ALIAS_READ_ONLY) { + // Theoretically, a user with privileges for index_aw* could write into alias_ab2w, as the write index is index_aw1 + // However, the index resolution code is not aware that this is a write operation; thus it resolves + // to all alias members which contain also index_bw1, for which we do not have permissions + assertThat(httpResponse, containsExactly().at("items[*].index[?(@.result == 'created')]._index")); + } else if (user != LIMITED_USER_NONE) { + assertThat( + httpResponse, + containsExactly(index_aw1).at("items[*].index[?(@.result == 'created')]._index") + .reducedBy(user.reference(WRITE)) + .whenEmpty(isOk()) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete("index_aw1/_doc/put_doc_alias_bulk_test_1"); + } + } + + @Test + public void putDocument_noExistingIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.put("index_bwx1/_doc/put_doc_non_existing_index_test_1", json("a, 1")); + assertThat( + httpResponse, + containsExactly(index_bwx1).at("_index").reducedBy(user.reference(CREATE_INDEX)).whenEmpty(isForbidden()) + ); + } finally { + delete(index_bwx1); + } + } + + @Test + public void createIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson("index_bwx1", "{}"); + assertThat( + httpResponse, + containsExactly(index_bwx1).at("index").reducedBy(user.reference(CREATE_INDEX)).whenEmpty(isForbidden()) + ); + } finally { + delete(index_bwx1); + } + } + + @Test + public void createIndex_systemIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson(".system_index_plugin_not_existing", "{}"); + + if (user.reference(CREATE_INDEX).covers(system_index_plugin_not_existing)) { + assertThat(httpResponse, isOk()); + } else if (user == SUPER_UNLIMITED_USER || (user == UNLIMITED_USER && !clusterConfig.systemIndexPrivilegeEnabled)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(system_index_plugin_not_existing); + } + } + + @Test + public void deleteIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_bwx1); + + HttpResponse httpResponse = restClient.delete("index_bwx1"); + if (user.reference(MANAGE_INDEX).covers(index_bwx1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(index_bwx1); + } + } + + @Test + public void deleteIndex_systemIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(system_index_plugin_not_existing); + + HttpResponse httpResponse = restClient.delete(".system_index_plugin_not_existing"); + + if (clusterConfig.systemIndexPrivilegeEnabled && user.reference(MANAGE_INDEX).covers(system_index_plugin_not_existing)) { + assertThat(httpResponse, isOk()); + } else if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(system_index_plugin_not_existing); + } + } + + @Test + public void createIndex_withAlias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.putJson("index_bwx1", """ + { + "aliases": { + "alias_bwx": {} + } + }"""); + + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.reference(MANAGE_ALIAS).covers(index_bwx1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + delete(index_bwx1); + } + } + + @Test + public void deleteAlias_staticIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(alias_bwx.on(index_bw1)); + + HttpResponse httpResponse = restClient.delete("index_bw1/_aliases/alias_bwx"); + + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.reference(MANAGE_ALIAS).covers(index_bw1) || user.reference(MANAGE_ALIAS).covers(alias_bwx)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + delete(alias_bwx); + } + } + + @Test + public void deleteAlias_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(alias_bwx.on(index_bw1)); + + HttpResponse httpResponse = restClient.delete("*/_aliases/alias_bwx"); + + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + // For all non super admin users, this will be rejected by SystemIndexAccessEvaluator: + // WARN SystemIndexAccessEvaluator:361 - indices:admin/aliases for '_all' indices is not allowed for a regular user + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(alias_bwx); + } + } + + @Test + public void aliases_createAlias() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + + HttpResponse httpResponse = restClient.postJson("_aliases", """ + { + "actions": [ + { "add": { "index": "index_bw1", "alias": "alias_bwx" } } + ] + }"""); + + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.reference(MANAGE_ALIAS).covers(index_bw1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + + } finally { + delete(alias_bwx); + } + } + + @Test + public void aliases_createAlias_indexPattern() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.postJson("_aliases", """ + { + "actions": [ + { "add": { "indices": ["index_bw*"], "alias": "alias_bwx" } } + ] + }"""); + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.reference(MANAGE_ALIAS).coversAll(index_bw1, index_bw2)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + delete(alias_bwx); + } + } + + @Test + public void aliases_deleteAlias_staticIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(alias_bwx.on(index_bw1)); + + HttpResponse httpResponse = restClient.postJson("_aliases", """ + { + "actions": [ + { "remove": { "index": "index_bw1", "alias": "alias_bwx" } } + ] + }"""); + + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.reference(MANAGE_ALIAS).covers(index_bw1) || user.reference(MANAGE_ALIAS).covers(alias_bwx)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + delete(alias_bwx); + } + } + + @Test + public void aliases_deleteAlias_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(alias_bwx.on(index_bw1, index_bw2)); + + HttpResponse httpResponse = restClient.postJson("_aliases", """ + { + "actions": [ + { "remove": { "index": "*", "alias": "alias_bwx" } } + ] + }"""); + + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + // For all non super admin users, this will be rejected by SystemIndexAccessEvaluator: + // WARN SystemIndexAccessEvaluator:361 - indices:admin/aliases for '_all' indices is not allowed for a regular user + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(alias_bwx); + } + } + + @Test + public void aliases_removeIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_bwx1); + + HttpResponse httpResponse = restClient.postJson("_aliases", """ + { + "actions": [ + { "remove_index": { "index": "index_bwx1" } } + ] + }"""); + + if (user.reference(MANAGE_INDEX).covers(index_bwx1)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete(index_bwx1); + } + } + + @Test + public void reindex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.postJson("_reindex", """ + { + "source": { "index": "index_br1" }, + "dest": { "index": "index_bwx1" } + }"""); + if (containsExactly(index_bwx1).reducedBy(user.reference(CREATE_INDEX)).isEmpty()) { + assertThat(httpResponse, isForbidden()); + assertThat(cluster.getAdminCertRestClient().get("index_bwx1/_search"), isNotFound()); + } else { + assertThat(httpResponse, isOk()); + assertThat(cluster.getAdminCertRestClient().get("index_bwx1/_search"), isOk()); + } + } finally { + delete(index_bwx1); + } + } + + @Test + public void cloneIndex() throws Exception { + String sourceIndex = "index_bw1"; + String targetIndex = "index_bwx1"; + + Client client = cluster.getInternalNodeClient(); + client.admin() + .indices() + .updateSettings(new UpdateSettingsRequest(sourceIndex).settings(Settings.builder().put("index.blocks.write", true).build())) + .actionGet(); + + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.post(sourceIndex + "/_clone/" + targetIndex); + assertThat( + httpResponse, + containsExactly(index_bwx1).at("index").reducedBy(user.reference(MANAGE_INDEX)).whenEmpty(isForbidden()) + ); + } finally { + cluster.getInternalNodeClient() + .admin() + .indices() + .updateSettings( + new UpdateSettingsRequest(sourceIndex).settings(Settings.builder().put("index.blocks.write", false).build()) + ) + .actionGet(); + delete(index_bwx1); + } + } + + @Test + public void closeIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.post("index_bw1/_close"); + assertThat( + httpResponse, + containsExactly(index_bw1).at("indices.keys()").reducedBy(user.reference(MANAGE_INDEX)).whenEmpty(isForbidden()) + ); + } finally { + cluster.getInternalNodeClient().admin().indices().open(new OpenIndexRequest("index_bw1")).actionGet(); + } + } + + @Test + public void closeIndex_wildcard() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.post("*/_close"); + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + // For all non super admin users, this will be rejected by SystemIndexAccessEvaluator: + // WARN SystemIndexAccessEvaluator:361 - indices:admin/close for '_all' indices is not allowed for a regular user + assertThat(httpResponse, isForbidden()); + } + } finally { + cluster.getInternalNodeClient().admin().indices().open(new OpenIndexRequest("*")).actionGet(); + } + } + + @Test + public void closeIndex_openIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + HttpResponse httpResponse = restClient.post("index_bw1/_close"); + assertThat( + httpResponse, + containsExactly(index_bw1).at("indices.keys()").reducedBy(user.reference(MANAGE_INDEX)).whenEmpty(isForbidden()) + ); + httpResponse = restClient.post("index_bw1/_open"); + + if (containsExactly(index_bw1).reducedBy(user.reference(MANAGE_INDEX)).isEmpty()) { + assertThat(httpResponse, isForbidden()); + } else { + assertThat(httpResponse, isOk()); + } + } finally { + cluster.getInternalNodeClient().admin().indices().open(new OpenIndexRequest("index_bw1")).actionGet(); + } + } + + @Test + public void rollover_explicitTargetIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(alias_bwx.on(index_bw1).writeIndex(index_bw1)); + + HttpResponse httpResponse = restClient.postJson("alias_bwx/_rollover/index_bwx1", """ + { + "conditions": { + "max_age": "0s" + } + }"""); + + if (clusterConfig.legacyPrivilegeEvaluation) { + if (user.reference(MANAGE_ALIAS).covers(index_bw1) && user.reference(MANAGE_INDEX).covers(index_bw2)) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + } + } finally { + delete(alias_bwx, index_bwx1); + } + } + + @After + public void refresh() { + cluster.getInternalNodeClient().admin().indices().refresh(new RefreshRequest("*")).actionGet(); + } + + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") + public static Collection params() { + List result = new ArrayList<>(); + + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + for (TestSecurityConfig.User user : USERS) { + result.add(new Object[] { clusterConfig, user, user.getDescription() }); + } + } + return result; + } + + public IndexAuthorizationReadWriteIntTests(ClusterConfig clusterConfig, TestSecurityConfig.User user, String description) + throws Exception { + this.user = user; + this.cluster = clusterConfig.cluster(IndexAuthorizationReadWriteIntTests::clusterBuilder); + this.clusterConfig = clusterConfig; + } + + private void createInitialTestObjects(TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + TestIndexOrAliasOrDatastream.createInitialTestObjects(cluster, testIndexOrAliasOrDatastreamArray); + } + + private void delete(TestIndexOrAliasOrDatastream... testIndexOrAliasOrDatastreamArray) { + TestIndexOrAliasOrDatastream.delete(cluster, testIndexOrAliasOrDatastreamArray); + } + + private void delete(String... paths) { + try (TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + for (String path : paths) { + HttpResponse response = adminRestClient.delete(path); + if (response.getStatusCode() != 200 && response.getStatusCode() != 404) { + throw new RuntimeException("Error while deleting " + path + "\n" + response.getBody()); + } + } + } + } +} diff --git a/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java new file mode 100644 index 0000000000..65ec63c271 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/security/privileges/int_tests/SnapshotAuthorizationIntTests.java @@ -0,0 +1,387 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.privileges.int_tests; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.google.common.collect.ImmutableList; +import org.apache.hc.core5.http.HttpEntity; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.action.admin.indices.refresh.RefreshRequest; +import org.opensearch.test.framework.TestSecurityConfig; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; +import org.opensearch.test.framework.matcher.RestIndexMatchers; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.cluster.TestRestClient.json; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnResponseIndexMatcher.containsExactly; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedTo; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.limitedToNone; +import static org.opensearch.test.framework.matcher.RestIndexMatchers.OnUserIndexMatcher.unlimitedIncludingOpenSearchSecurityIndex; +import static org.opensearch.test.framework.matcher.RestMatchers.isForbidden; +import static org.opensearch.test.framework.matcher.RestMatchers.isOk; + +/** + * TODO requests on non cm node + */ +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class SnapshotAuthorizationIntTests { + static final TestIndex index_a1 = TestIndex.name("index_ar1").documentCount(10).seed(1).build(); + static final TestIndex index_a2 = TestIndex.name("index_ar2").documentCount(11).seed(2).build(); + static final TestIndex index_a3 = TestIndex.name("index_ar3").documentCount(12).seed(3).build(); + static final TestIndex index_b1 = TestIndex.name("index_br1").documentCount(4).seed(4).build(); + static final TestIndex index_b2 = TestIndex.name("index_br2").documentCount(5).seed(5).build(); + static final TestIndex index_b3 = TestIndex.name("index_br3").documentCount(6).seed(6).build(); + + static final TestIndex system_index_plugin_not_existing = TestIndex.name(".system_index_plugin_not_existing") + .hidden() + .documentCount(0) + .build(); // not initially created + + static final TestIndex index_awx1 = TestIndex.name("index_awx1").documentCount(10).seed(11).build(); // not initially created + static final TestIndex index_awx2 = TestIndex.name("index_awx2").documentCount(10).seed(12).build(); // not initially created + + static final TestIndex index_bwx1 = TestIndex.name("index_bwx1").documentCount(10).seed(13).build(); // not initially created + static final TestIndex index_bwx2 = TestIndex.name("index_bwx2").documentCount(10).seed(14).build(); // not initially created + + /** + * This key identifies assertion reference data for index search/read permissions of individual users. + */ + static final TestSecurityConfig.User.MetadataKey READ = new TestSecurityConfig.User.MetadataKey<>( + "read", + RestIndexMatchers.IndexMatcher.class + ); + + /** + * This key identifies assertion reference data for index write permissions of individual users. This does + * not include index creation permissions. + */ + static final TestSecurityConfig.User.MetadataKey WRITE = new TestSecurityConfig.User.MetadataKey<>( + "write", + RestIndexMatchers.IndexMatcher.class + ); + + static TestSecurityConfig.User LIMITED_USER_A = new TestSecurityConfig.User("limited_user_A")// + .description("index_a*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor", "manage_snapshots")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_a*")// + .indexPermissions("write", "manage") + .on("index_aw*") + )// + .reference(READ, limitedTo(index_a1, index_a2, index_awx1, index_awx2))// + .reference(WRITE, limitedTo(index_awx1, index_awx2)); + + static TestSecurityConfig.User LIMITED_USER_B = new TestSecurityConfig.User("limited_user_B")// + .description("index_b*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor", "manage_snapshots")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_b*")// + .indexPermissions("write", "manage") + .on("index_bw*") + )// + .reference(READ, limitedTo(index_b1, index_b2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bwx1, index_bwx2)); + + static TestSecurityConfig.User LIMITED_USER_B_SYSTEM_INDEX = new TestSecurityConfig.User("limited_user_B_system_index")// + .description("index_b*, .system_index_plugin")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor", "manage_snapshots")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_b*")// + .indexPermissions("write", "manage") + .on("index_bw*") + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*", "system:admin/system_index") + .on(".system_index_plugin", ".system_index_plugin_not_existing") + .indexPermissions("write", "manage", "system:admin/system_index") + .on(".system_index_plugin_not_existing") + + )// + .reference(READ, limitedTo(index_b1, index_b2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_bwx1, index_bwx2, system_index_plugin_not_existing)); + + static TestSecurityConfig.User LIMITED_USER_AB = new TestSecurityConfig.User("limited_user_AB")// + .description("index_a*, index_b*")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops", "cluster_monitor", "manage_snapshots")// + .indexPermissions("read", "indices_monitor", "indices:admin/refresh*") + .on("index_a*", "index_b*")// + .indexPermissions("write", "manage") + .on("index_aw*", "index_bw*") + )// + .reference(READ, limitedTo(index_a1, index_a2, index_awx1, index_awx2, index_b1, index_b2, index_bwx1, index_bwx2))// + .reference(WRITE, limitedTo(index_awx1, index_awx2, index_bwx1, index_bwx2)); + + static final TestSecurityConfig.User LIMITED_USER_NONE = new TestSecurityConfig.User("limited_user_none")// + .description("no index privileges")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor") + )// + .reference(READ, limitedToNone())// + .reference(WRITE, limitedToNone()); + + static final TestSecurityConfig.User UNLIMITED_USER = new TestSecurityConfig.User("unlimited_user")// + .description("unlimited")// + .roles( + new TestSecurityConfig.Role("r1")// + .clusterPermissions("cluster_composite_ops_ro", "cluster_monitor", "manage_snapshots") + .indexPermissions("*") + .on("*")// + + )// + .reference( + READ, + limitedTo(index_a1, index_a2, index_a3, index_awx1, index_awx2, index_b1, index_b2, index_b3, index_bwx1, index_bwx2) + )// + .reference( + WRITE, + limitedTo(index_a1, index_a2, index_a3, index_awx1, index_awx2, index_b1, index_b2, index_b3, index_bwx1, index_bwx2) + ); + + /** + * The SUPER_UNLIMITED_USER authenticates with an admin cert, which will cause all access control code to be skipped. + * This serves as a base for comparison with the default behavior. + */ + static final TestSecurityConfig.User SUPER_UNLIMITED_USER = new TestSecurityConfig.User("super_unlimited_user")// + .description("super unlimited (admin cert)")// + .adminCertUser()// + .reference(READ, unlimitedIncludingOpenSearchSecurityIndex())// + .reference(WRITE, unlimitedIncludingOpenSearchSecurityIndex()); + + static final List USERS = ImmutableList.of( + LIMITED_USER_A, + LIMITED_USER_B, + LIMITED_USER_B_SYSTEM_INDEX, + LIMITED_USER_AB, + LIMITED_USER_NONE, + UNLIMITED_USER, + SUPER_UNLIMITED_USER + ); + + static LocalCluster.Builder clusterBuilder() { + return new LocalCluster.Builder().singleNode() + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USERS)// + .indices(index_a1, index_a2, index_a3, index_b1, index_b2, index_b3)// + .snapshotRepositories("test_repository") + .plugin(IndexAuthorizationReadOnlyIntTests.SystemIndexTestPlugin.class); + } + + @AfterClass + public static void stopClusters() { + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + clusterConfig.shutdown(); + } + } + + final TestSecurityConfig.User user; + final LocalCluster cluster; + final ClusterConfig clusterConfig; + + @Test + public void restore_singleIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1); + createInitialTestSnapshot("_snapshot/test_repository/single_index_snapshot", json("indices", "index_awx1")); + + delete(index_awx1); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/single_index_snapshot/_restore?wait_for_completion=true" + ); + + assertThat(httpResponse, containsExactly(index_awx1).at("snapshot.indices").butForbiddenIfIncomplete(user.reference(WRITE))); + + } finally { + delete("_snapshot/test_repository/single_index_snapshot"); + delete(index_awx1); + } + } + + @Test + public void restore_singleIndex_rename1() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1); + createInitialTestSnapshot("_snapshot/test_repository/single_index_snapshot", json("indices", "index_awx1")); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/single_index_snapshot/_restore?wait_for_completion=true", + json("rename_pattern", "index_(.+)x1", "rename_replacement", "index_$1x2") + ); + + assertThat(httpResponse, containsExactly(index_awx2).at("snapshot.indices").butForbiddenIfIncomplete(user.reference(WRITE))); + + } finally { + delete("_snapshot/test_repository/single_index_snapshot"); + delete(index_awx1, index_awx2); + } + } + + @Test + public void restore_singleIndex_rename2() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1); + createInitialTestSnapshot("_snapshot/test_repository/single_index_snapshot", json("indices", "index_awx1")); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/single_index_snapshot/_restore?wait_for_completion=true", + json("rename_pattern", "index_a(.*)", "rename_replacement", "index_b$1") + ); + + assertThat(httpResponse, containsExactly(index_bwx1).at("snapshot.indices").butForbiddenIfIncomplete(user.reference(WRITE))); + + } finally { + delete("_snapshot/test_repository/single_index_snapshot"); + delete(index_awx1, index_bwx1); + } + } + + @Test + public void restore_singleIndex_renameToSystemIndex() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1); + createInitialTestSnapshot("_snapshot/test_repository/single_index_snapshot", json("indices", "index_awx1")); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/single_index_snapshot/_restore?wait_for_completion=true", + json("rename_pattern", "index_awx1", "rename_replacement", system_index_plugin_not_existing.name()) + ); + + if (clusterConfig.systemIndexPrivilegeEnabled || user == SUPER_UNLIMITED_USER) { + assertThat( + httpResponse, + containsExactly(system_index_plugin_not_existing).at("snapshot.indices").butForbiddenIfIncomplete(user.reference(WRITE)) + ); + } else { + assertThat(httpResponse, isForbidden()); + } + } finally { + delete("_snapshot/test_repository/single_index_snapshot"); + delete(index_awx1, system_index_plugin_not_existing); + } + } + + @Test + public void restore_singleIndexFromAllIndices() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1); + createInitialTestSnapshot("_snapshot/test_repository/all_index_snapshot", json()); + + delete(index_awx1); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/all_index_snapshot/_restore?wait_for_completion=true", + json("indices", "index_awx1") + ); + + assertThat(httpResponse, containsExactly(index_awx1).at("snapshot.indices").butForbiddenIfIncomplete(user.reference(WRITE))); + + } finally { + delete("_snapshot/test_repository/all_index_snapshot"); + delete(index_awx1); + } + } + + @Test + public void restore_all_globalState() throws Exception { + try (TestRestClient restClient = cluster.getRestClient(user)) { + createInitialTestObjects(index_awx1, index_awx2, index_bwx1, index_bwx2); + createInitialTestSnapshot("_snapshot/test_repository/all_index_snapshot", json("indices", "index_*w*")); + + delete(index_awx1, index_awx2, index_bwx1, index_bwx2); + + TestRestClient.HttpResponse httpResponse = restClient.post( + "_snapshot/test_repository/all_index_snapshot/_restore?wait_for_completion=true", + json("include_global_state", true) + ); + + if (user == SUPER_UNLIMITED_USER) { + assertThat(httpResponse, isOk()); + } else { + assertThat(httpResponse, isForbidden()); + } + + } finally { + delete("_snapshot/test_repository/all_index_snapshot"); + delete(index_awx1, index_awx2, index_bwx1, index_bwx2); + } + } + + @After + public void refresh() { + cluster.getInternalNodeClient().admin().indices().refresh(new RefreshRequest("*")).actionGet(); + } + + @ParametersFactory(shuffle = false, argumentFormatting = "%1$s, %3$s") + public static Collection params() { + List result = new ArrayList<>(); + + for (ClusterConfig clusterConfig : ClusterConfig.values()) { + for (TestSecurityConfig.User user : USERS) { + result.add(new Object[] { clusterConfig, user, user.getDescription() }); + } + } + return result; + } + + public SnapshotAuthorizationIntTests(ClusterConfig clusterConfig, TestSecurityConfig.User user, String description) throws Exception { + this.user = user; + this.cluster = clusterConfig.cluster(SnapshotAuthorizationIntTests::clusterBuilder); + this.clusterConfig = clusterConfig; + } + + private void createInitialTestObjects(TestIndexOrAliasOrDatastream... testIndexLikeArray) { + TestIndexOrAliasOrDatastream.createInitialTestObjects(cluster, testIndexLikeArray); + } + + private void createInitialTestSnapshot(String snapshotName, HttpEntity requestBody) { + try (TestRestClient client = cluster.getAdminCertRestClient()) { + TestRestClient.HttpResponse httpResponse = client.put(snapshotName + "?wait_for_completion=true", requestBody); + assertThat(httpResponse, isOk()); + } + } + + private void delete(TestIndexOrAliasOrDatastream... testIndexLikeArray) { + TestIndexOrAliasOrDatastream.delete(cluster, testIndexLikeArray); + } + + private void delete(String... paths) { + try (TestRestClient adminRestClient = cluster.getAdminCertRestClient()) { + for (String path : paths) { + TestRestClient.HttpResponse response = adminRestClient.delete(path); + if (response.getStatusCode() != 200 && response.getStatusCode() != 404) { + throw new RuntimeException("Error while deleting " + path + "\n" + response.getBody()); + } + } + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java b/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java deleted file mode 100644 index 734e0b5333..0000000000 --- a/src/integrationTest/java/org/opensearch/test/framework/TestIndex.java +++ /dev/null @@ -1,94 +0,0 @@ -/* -* Copyright 2021-2022 floragunn GmbH -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -*/ - -/* -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -* -* Modifications Copyright OpenSearch Contributors. See -* GitHub history for details. -*/ - -package org.opensearch.test.framework; - -import org.opensearch.action.admin.indices.create.CreateIndexRequest; -import org.opensearch.common.settings.Settings; -import org.opensearch.transport.client.Client; - -public class TestIndex { - - private final String name; - private final Settings settings; - private final TestData testData; - - public TestIndex(String name, Settings settings, TestData testData) { - this.name = name; - this.settings = settings; - this.testData = testData; - } - - public void create(Client client) { - if (testData != null) { - testData.createIndex(client, name, settings); - } else { - client.admin().indices().create(new CreateIndexRequest(name).settings(settings)).actionGet(); - } - } - - public String name() { - return name; - } - - public static Builder name(String name) { - return new Builder().name(name); - } - - public static class Builder { - private String name; - private Settings.Builder settings = Settings.builder(); - private TestData testData; - - public Builder name(String name) { - this.name = name; - return this; - } - - public Builder setting(String name, int value) { - settings.put(name, value); - return this; - } - - public Builder shards(int value) { - settings.put("index.number_of_shards", 5); - return this; - } - - public Builder data(TestData testData) { - this.testData = testData; - return this; - } - - public TestIndex build() { - return new TestIndex(name, settings.build(), testData); - } - - } - -} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java index d58070ab45..4f94af9796 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java +++ b/src/integrationTest/java/org/opensearch/test/framework/TestSecurityConfig.java @@ -72,6 +72,8 @@ import org.opensearch.security.securityconf.impl.v7.RoleV7; import org.opensearch.security.support.ConfigConstants; import org.opensearch.test.framework.cluster.OpenSearchClientProvider.UserCredentialsHolder; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.matcher.RestIndexMatchers; import org.opensearch.transport.client.Client; import static org.apache.http.HttpHeaders.AUTHORIZATION; @@ -325,7 +327,6 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params if (doNotFailOnForbidden != null) { xContentBuilder.field("do_not_fail_on_forbidden", doNotFailOnForbidden); } - xContentBuilder.field("authc", authcDomainMap); if (authzDomainMap.isEmpty() == false) { xContentBuilder.field("authz", authzDomainMap); @@ -459,6 +460,8 @@ public static final class User implements UserCredentialsHolder, ToXContentObjec String requestedTenant; private Map attributes = new HashMap<>(); private Map, Object> matchers = new HashMap<>(); + private Map indexMatchers = new HashMap<>(); + private boolean adminCertUser = false; private Boolean hidden = null; @@ -538,6 +541,20 @@ public Set getRoleNames() { return roles.stream().map(Role::getName).collect(Collectors.toSet()); } + public String getDescription() { + return description; + } + + @Override + public boolean isAdminCertUser() { + return adminCertUser; + } + + public User adminCertUser() { + this.adminCertUser = true; + return this; + } + public Object getAttribute(String attributeName) { return attributes.get(attributeName); } @@ -551,7 +568,7 @@ public T reference(MetadataKey key) { if (result != null) { return key.type.cast(result); } else { - return null; + throw new RuntimeException("Unknown reference " + key + " in user " + this.name); } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java index c1e6fca059..5aef9f29b4 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/LocalCluster.java @@ -44,6 +44,7 @@ import org.apache.logging.log4j.Logger; import org.junit.rules.ExternalResource; +import org.opensearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.opensearch.common.settings.Settings; import org.opensearch.node.PluginAwareNode; import org.opensearch.plugins.Plugin; @@ -57,13 +58,17 @@ import org.opensearch.test.framework.AuthFailureListeners; import org.opensearch.test.framework.AuthzDomain; import org.opensearch.test.framework.OnBehalfOfConfig; -import org.opensearch.test.framework.TestIndex; import org.opensearch.test.framework.TestSecurityConfig; import org.opensearch.test.framework.TestSecurityConfig.Role; import org.opensearch.test.framework.XffConfig; import org.opensearch.test.framework.audit.TestRuleAuditLogSink; import org.opensearch.test.framework.certificate.CertificateData; import org.opensearch.test.framework.certificate.TestCertificates; +import org.opensearch.test.framework.data.TestAlias; +import org.opensearch.test.framework.data.TestComponentTemplate; +import org.opensearch.test.framework.data.TestDataStream; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexTemplate; import org.opensearch.transport.client.Client; /** @@ -98,6 +103,11 @@ public class LocalCluster extends ExternalResource implements AutoCloseable, Ope private final Map remotes; private volatile LocalOpenSearchCluster localOpenSearchCluster; private final List testIndices; + private final List testAliases; + private final List testDataStreams; + private final List testComponentTemplates; + private final List testIndexTemplates; + private final List testSnapshotRepositories; private boolean loadConfigurationIntoIndex; @@ -114,6 +124,11 @@ private LocalCluster( List clusterDependencies, Map remotes, List testIndices, + List testAliases, + List testDataStreams, + List testComponentTemplates, + List testIndexTemplates, + List testSnapshotRepositories, boolean loadConfigurationIntoIndex, String defaultConfigurationInitDirectory, Integer expectedNodeStartupCount @@ -131,6 +146,11 @@ private LocalCluster( this.remotes = remotes; this.clusterDependencies = clusterDependencies; this.testIndices = testIndices; + this.testAliases = testAliases; + this.testDataStreams = testDataStreams; + this.testComponentTemplates = testComponentTemplates; + this.testIndexTemplates = testIndexTemplates; + this.testSnapshotRepositories = testSnapshotRepositories; this.loadConfigurationIntoIndex = loadConfigurationIntoIndex; if (StringUtils.isNoneBlank(defaultConfigurationInitDirectory)) { System.setProperty(INIT_CONFIGURATION_DIR, defaultConfigurationInitDirectory); @@ -263,11 +283,29 @@ private void start() { } try (Client client = getInternalNodeClient()) { + for (TestComponentTemplate testComponentTemplate : this.testComponentTemplates) { + testComponentTemplate.create(client); + } + for (TestIndexTemplate indexTemplate : this.testIndexTemplates) { + indexTemplate.create(client); + } + for (TestIndex index : this.testIndices) { index.create(client); } - } + for (TestDataStream dataStream : this.testDataStreams) { + dataStream.create(client); + } + + for (TestAlias alias : this.testAliases) { + alias.create(client); + } + + for (String snapshotRepository : this.testSnapshotRepositories) { + createSnapshotRepository(client, snapshotRepository); + } + } } catch (Exception e) { log.error("Local ES cluster start failed", e); throw new RuntimeException(e); @@ -319,6 +357,13 @@ public void triggerConfigurationReloadForCTypes(Client client, List cType } } + private void createSnapshotRepository(Client client, String snapshotRepository) { + client.admin() + .cluster() + .putRepository(new PutRepositoryRequest(snapshotRepository).type("fs").settings(Map.of("location", getSnapshotDirPath()))) + .actionGet(); + } + public CertificateData getAdminCertificate() { return testCertificates.getAdminCertificateData(); } @@ -335,6 +380,11 @@ public static class Builder { private Map remoteClusters = new HashMap<>(); private List clusterDependencies = new ArrayList<>(); private List testIndices = new ArrayList<>(); + private List testAliases = new ArrayList<>(); + private List testDataStreams = new ArrayList<>(); + private List testIndexTemplates = new ArrayList<>(); + private List testComponentTemplates = new ArrayList<>(); + private List testSnapshotRepositories = new ArrayList<>(); private ClusterManager clusterManager = ClusterManager.DEFAULT; private TestSecurityConfig testSecurityConfig = new TestSecurityConfig(); private String clusterName = "local_cluster"; @@ -472,6 +522,33 @@ public Builder indices(Collection indices) { return this; } + public Builder aliases(TestAlias... aliases) { + this.testAliases.addAll(Arrays.asList(aliases)); + return this; + } + + public Builder dataStreams(TestDataStream... dataStreams) { + this.testDataStreams.addAll(Arrays.asList(dataStreams)); + return this; + } + + public Builder indexTemplates(TestIndexTemplate... indexTemplates) { + for (TestIndexTemplate indexTemplate : indexTemplates) { + this.testIndexTemplates.add(indexTemplate); + for (TestComponentTemplate testComponentTemplate : indexTemplate.getComposedOf()) { + if (!this.testComponentTemplates.contains(testComponentTemplate)) { + this.testComponentTemplates.add(testComponentTemplate); + } + } + } + return this; + } + + public Builder snapshotRepositories(String... repositoryNames) { + this.testSnapshotRepositories.addAll(Arrays.asList(repositoryNames)); + return this; + } + public Builder users(TestSecurityConfig.User... users) { return this.users(Arrays.asList(users)); } @@ -610,6 +687,11 @@ public LocalCluster build() { clusterDependencies, remoteClusters, testIndices, + testAliases, + testDataStreams, + testComponentTemplates, + testIndexTemplates, + testSnapshotRepositories, loadConfigurationIntoIndex, defaultConfigurationInitDirectory, expectedNodeStartupCount diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java index 131ff65615..5c3340ff2a 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/OpenSearchClientProvider.java @@ -105,9 +105,16 @@ default TestRestClient getRestClient(UserCredentialsHolder user, CertificateData } default TestRestClient getRestClient(UserCredentialsHolder user, Header... headers) { + if (user.isAdminCertUser()) { + return getRestClient(getTestCertificates().getAdminCertificateData()); + } return getRestClient(user.getName(), user.getPassword(), null, headers); } + default TestRestClient getAdminCertRestClient() { + return getRestClient(getTestCertificates().getAdminCertificateData()); + } + default RestHighLevelClient getRestHighLevelClient(String username, String password, Header... headers) { return getRestHighLevelClient(new UserCredentialsHolder() { @Override @@ -298,6 +305,10 @@ public interface UserCredentialsHolder { String getName(); String getPassword(); + + default boolean isAdminCertUser() { + return false; + } } } diff --git a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java index fd4acbfb76..ec0f79e73c 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java +++ b/src/integrationTest/java/org/opensearch/test/framework/cluster/TestRestClient.java @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -59,6 +60,7 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.client5.http.routing.HttpRoutePlanner; +import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.io.entity.StringEntity; @@ -72,9 +74,13 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.security.DefaultObjectMapper; +import com.nimbusds.jose.shaded.gson.Gson; + import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -120,6 +126,12 @@ public HttpResponse get(String path, Header... headers) { return executeRequest(new HttpGet(getHttpServerUri() + "/" + path), headers); } + public HttpResponse get(String path, HttpEntity entity, Header... headers) { + HttpGet uriRequest = new HttpGet(getHttpServerUri() + "/" + path); + uriRequest.setEntity(entity); + return executeRequest(uriRequest, headers); + } + public HttpResponse getWithoutLeadingSlash(String path, Header... headers) { HttpUriRequest req = new HttpGet(getHttpServerUri()); req.setPath(path); @@ -184,10 +196,22 @@ public HttpResponse put(String path) { return executeRequest(uriRequest); } + public HttpResponse put(String path, HttpEntity entity, Header... headers) { + HttpPut uriRequest = new HttpPut(getHttpServerUri() + "/" + path); + uriRequest.setEntity(entity); + return executeRequest(uriRequest, headers); + } + public HttpResponse delete(String path, Header... headers) { return executeRequest(new HttpDelete(getHttpServerUri() + "/" + path), headers); } + public HttpResponse delete(String path, HttpEntity entity, Header... headers) { + HttpDelete uriRequest = new HttpDelete(getHttpServerUri() + "/" + path); + uriRequest.setEntity(entity); + return executeRequest(uriRequest, headers); + } + public HttpResponse postJson(String path, String body, Header... headers) { HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); uriRequest.setEntity(new StringEntity(body)); @@ -203,6 +227,12 @@ public HttpResponse post(String path) { return executeRequest(uriRequest); } + public HttpResponse post(String path, HttpEntity entity, Header... headers) { + HttpPost uriRequest = new HttpPost(getHttpServerUri() + "/" + path); + uriRequest.setEntity(entity); + return executeRequest(uriRequest, headers); + } + public HttpResponse patch(String path, ToXContentObject body) { return patch(path, Strings.toString(XContentType.JSON, body)); } @@ -315,12 +345,12 @@ public HttpResponse(CloseableHttpResponse inner) throws IllegalStateException, I private void verifyContentType() { final String contentType = this.getHeader(HttpHeaders.CONTENT_TYPE).getValue(); if (contentType.contains("application/json")) { - assertThat("Response body format was not json, body: " + body, body.charAt(0), equalTo('{')); + assertThat("Response body format was not json, body: " + body, body.charAt(0), anyOf(equalTo('{'), equalTo('['))); } else { assertThat( "Response body format was json, whereas content-type was " + contentType + ", body: " + body, body.charAt(0), - not(equalTo('{')) + allOf(not(equalTo('{')), not(equalTo('['))) ); } @@ -495,4 +525,19 @@ public void close() { // TODO: Is there anything to clean up here? } + /** + * Helper method to create very simple dynamic JSON request bodies. + * @param attributes Key-value pairs, keys on even indices, values on odd indices. + * @return A request body that can be passed to the get(), post(), etc. methods. + */ + public static HttpEntity json(Object... attributes) { + Map map = new HashMap<>(); + + for (int i = 0; i < attributes.length - 1; i += 2) { + map.put(attributes[i].toString(), attributes[i + 1]); + } + + return new StringEntity(new Gson().toJson(map), ContentType.APPLICATION_JSON); + } + } diff --git a/src/integrationTest/java/org/opensearch/test/framework/data/TestAlias.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestAlias.java new file mode 100644 index 0000000000..6b964b6b6b --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestAlias.java @@ -0,0 +1,153 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.data; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableSet; + +import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.opensearch.rest.action.admin.indices.AliasesNotFoundException; +import org.opensearch.transport.client.Client; + +public class TestAlias implements TestIndexOrAliasOrDatastream { + + private final String name; + private final ImmutableSet indices; + private final TestIndexOrAliasOrDatastream writeIndex; + private final boolean hidden; + + private Set documentIds; + private Map documents; + + public TestAlias(String name, TestIndexOrAliasOrDatastream... indices) { + this.name = name; + this.indices = ImmutableSet.copyOf(indices); + this.writeIndex = null; + this.hidden = false; + } + + TestAlias(String name, ImmutableSet indices, TestIndexOrAliasOrDatastream writeIndex, boolean hidden) { + this.name = name; + this.indices = indices; + this.writeIndex = writeIndex; + this.hidden = hidden; + } + + public TestAlias on(TestIndexOrAliasOrDatastream... indices) { + return new TestAlias(this.name, ImmutableSet.copyOf(indices), this.writeIndex, this.hidden); + } + + public TestAlias writeIndex(TestIndexOrAliasOrDatastream writeIndex) { + return new TestAlias(this.name, this.indices, writeIndex, this.hidden); + } + + public TestAlias hidden() { + return new TestAlias(this.name, this.indices, this.writeIndex, true); + } + + @Override + public String toString() { + return "Test alias '" + name + "'"; + } + + @Override + public void create(Client client) { + client.admin() + .indices() + .aliases( + new IndicesAliasesRequest().addAliasAction( + IndicesAliasesRequest.AliasActions.add().indices(getIndexNamesAsArray()).alias(name).isHidden(hidden) + ) + ) + .actionGet(); + + if (writeIndex != null) { + client.admin() + .indices() + .aliases( + new IndicesAliasesRequest().addAliasAction( + IndicesAliasesRequest.AliasActions.add().index(writeIndex.name()).alias(name).writeIndex(true) + ) + ) + .actionGet(); + } + } + + @Override + public void delete(Client client) { + try { + client.admin() + .indices() + .aliases(new IndicesAliasesRequest().addAliasAction(IndicesAliasesRequest.AliasActions.remove().alias(name).indices("*"))) + .actionGet(); + } catch (AliasesNotFoundException e) { + // It is fine if the alias to be deleted does not exist + } + } + + @Override + public String name() { + return name; + } + + public ImmutableSet getIndices() { + return indices; + } + + public String[] getIndexNamesAsArray() { + return indices.stream().map(TestIndexOrAliasOrDatastream::name).collect(Collectors.toSet()).toArray(new String[0]); + } + + @Override + public Set documentIds() { + Set result = this.documentIds; + + if (result == null) { + result = new HashSet<>(); + for (TestIndexOrAliasOrDatastream testIndex : this.indices) { + result.addAll(testIndex.documentIds()); + } + + result = Collections.unmodifiableSet(result); + this.documentIds = result; + } + + return result; + } + + @Override + public Map documents() { + Map result = this.documents; + + if (result == null) { + result = new HashMap<>(); + for (TestIndexOrAliasOrDatastream testIndex : this.indices) { + result.putAll(testIndex.documents()); + } + + result = Collections.unmodifiableMap(result); + this.documents = result; + } + + return result; + } + + public static TestIndex.Builder name(String name) { + return new TestIndex.Builder().name(name); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/data/TestComponentTemplate.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestComponentTemplate.java new file mode 100644 index 0000000000..9beff16754 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestComponentTemplate.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.data; + +import java.util.Map; + +import com.google.common.collect.ImmutableMap; + +import org.opensearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.opensearch.cluster.metadata.ComponentTemplate; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.transport.client.Client; + +public class TestComponentTemplate { + public static TestComponentTemplate DATA_STREAM_MINIMAL = new TestComponentTemplate( + "test_component_template_data_stream_minimal", + new TestMapping(new TestMapping.Property("@timestamp", "date", "date_optional_time||epoch_millis")) + ); + + private final String name; + private final TestMapping mapping; + + public TestComponentTemplate(String name, TestMapping mapping) { + this.name = name; + this.mapping = mapping; + } + + public String getName() { + return name; + } + + public TestMapping getMapping() { + return mapping; + } + + public void create(Client client) throws Exception { + try (XContentBuilder builder = JsonXContent.contentBuilder().map(getAsMap())) { + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + BytesReference.bytes(builder).streamInput() + ) + ) { + client.admin() + .indices() + .execute( + PutComponentTemplateAction.INSTANCE, + new PutComponentTemplateAction.Request(name).componentTemplate(ComponentTemplate.parse(parser)) + ) + .actionGet(); + } + } + } + + public Map getAsMap() { + return ImmutableMap.of("template", ImmutableMap.of("mappings", mapping.getAsMap())); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/TestData.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestData.java similarity index 86% rename from src/integrationTest/java/org/opensearch/test/framework/TestData.java rename to src/integrationTest/java/org/opensearch/test/framework/data/TestData.java index 606b56c834..9f2f97496c 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/TestData.java +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestData.java @@ -8,9 +8,10 @@ * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ -package org.opensearch.test.framework; +package org.opensearch.test.framework.data; import java.nio.ByteBuffer; +import java.time.Instant; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; @@ -18,6 +19,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -27,13 +29,16 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.DocWriteRequest; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.refresh.RefreshRequest; +import org.opensearch.action.admin.indices.rollover.RolloverRequest; import org.opensearch.action.delete.DeleteRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.common.settings.Settings; @@ -88,6 +93,16 @@ public static TestData.Builder documentCount(int documentCount) { "attr_object.obj_attr_object.obj_obj_attr_text.keyword" ); + public static final ImmutableList DEPARTMENTS = ImmutableList.of( + "dept_a_1", + "dept_a_2", + "dept_a_3", + "dept_b_1", + "dept_b_2", + "dept_c", + "dept_d" + ); + private static final Cache cache; static { @@ -97,7 +112,6 @@ public static TestData.Builder documentCount(int documentCount) { private String[] ipAddresses; private String[] threeWordPhrases; - private String[] departments = new String[] { "dept_a_1", "dept_a_2", "dept_a_3", "dept_b_1", "dept_b_2", "dept_c", "dept_d" }; private int size; private int deletedDocumentCount; private int refreshAfter; @@ -106,45 +120,20 @@ public static TestData.Builder documentCount(int documentCount) { private Map> documentsByDepartment; private Set deletedDocuments; private long subRandomSeed; + private final String timestampColumn; - public TestData(int seed, int size, int deletedDocumentCount, int refreshAfter) { + public TestData(int seed, int size, int deletedDocumentCount, int refreshAfter, String timestampColumnName) { Random random = new Random(seed); this.ipAddresses = createRandomIpAddresses(random); this.threeWordPhrases = createRandomThreeWordPhrases(random); this.size = size; this.deletedDocumentCount = deletedDocumentCount; this.refreshAfter = refreshAfter; - // this.additionalAttributes = additionalAttributes; this.subRandomSeed = random.nextLong(); + this.timestampColumn = timestampColumnName; this.createTestDocuments(random); } - private TestData( - String[] ipAddresses, - String[] departments, - int size, - int deletedDocumentCount, - int refreshAfter, - Map allDocuments, - Map retainedDocuments, - Map> documentsByDepartment, - Set deletedDocuments, - long subRandomSeed - ) { - super(); - this.ipAddresses = ipAddresses; - this.departments = departments; - this.size = size; - this.deletedDocumentCount = deletedDocumentCount; - this.refreshAfter = refreshAfter; - this.allDocuments = allDocuments; - this.retainedDocuments = retainedDocuments; - this.documentsByDepartment = documentsByDepartment; - // this.additionalAttributes = additionalAttributes; - this.deletedDocuments = deletedDocuments; - this.subRandomSeed = subRandomSeed; - } - public void createIndex(Client client, String name, Settings settings) { log.info( "creating test index " @@ -220,6 +209,54 @@ public void createIndex(Client client, String name, Settings settings) { log.info("Test index creation finished after " + (System.currentTimeMillis() - start) + " ms"); } + public void putDocuments(Client client, String name, int rolloverAfter) { + try { + Random random = new Random(subRandomSeed); + long start = System.currentTimeMillis(); + + int nextRefresh = (int) Math.floor((random.nextGaussian() * 0.5 + 0.5) * refreshAfter); + int nextRollover = rolloverAfter != -1 ? rolloverAfter : Integer.MAX_VALUE; + int i = 0; + + for (Map.Entry entry : allDocuments.entrySet()) { + String id = entry.getKey(); + TestDocument document = entry.getValue(); + + client.index( + new IndexRequest(name).source(document.content, XContentType.JSON).id(id).opType(DocWriteRequest.OpType.CREATE) + ).actionGet(); + + if (i > nextRefresh) { + client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); + double g = random.nextGaussian(); + + nextRefresh = (int) Math.floor((g * 0.5 + 1) * refreshAfter) + i + 1; + log.debug("refresh at " + i + " " + g + " " + (g * 0.5 + 1)); + } + + if (i > nextRollover) { + // This creates several generations of backing indices for data streams + client.admin().indices().rolloverIndex(new RolloverRequest(name, null)); + + nextRollover += rolloverAfter; + } + + i++; + } + + client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); + + for (String id : deletedDocuments) { + client.delete(new DeleteRequest(name, id)).actionGet(); + } + + client.admin().indices().refresh(new RefreshRequest(name)).actionGet(); + log.info("Test index creation finished after " + (System.currentTimeMillis() - start) + " ms"); + } catch (Exception e) { + throw new RuntimeException("Error while wring test documents to index " + name, e); + } + } + private void createTestDocuments(Random random) { Map allDocuments = new HashMap<>(size); @@ -365,6 +402,9 @@ private TestDocument randomDocument(Random random) { ImmutableMap.of("obj_obj_attr_text", "value_" + random.nextInt()) ) ); + if (timestampColumn != null) { + builder.put(timestampColumn, randomTimestamp(random)); + } return new TestDocument(randomId(random), builder.build()); } @@ -374,7 +414,12 @@ private String randomIpAddress(Random random) { } private String randomDepartmentName(Random random) { - return departments[random.nextInt(departments.length)]; + return DEPARTMENTS.get(random.nextInt(DEPARTMENTS.size())); + } + + private String randomTimestamp(Random random) { + long epochMillis = random.longs(1, -2857691960709L, 2857691960709L).findFirst().getAsLong(); + return Instant.ofEpochMilli(epochMillis).toString(); } private String randomThreeWordPhrase(Random random) { @@ -418,6 +463,10 @@ public int getDeletedDocumentCount() { return deletedDocumentCount; } + public Map getRetainedDocuments() { + return retainedDocuments; + } + public TestDocuments documents() { return new TestDocuments(this.retainedDocuments); } @@ -442,15 +491,16 @@ private static class Key { private final int size; private final int deletedDocumentCount; private final int refreshAfter; - // private final ImmutableMap additionalAttributes; + private final String timestampColumnName; - public Key(int seed, int size, int deletedDocumentCount, int refreshAfter) { + public Key(int seed, int size, int deletedDocumentCount, int refreshAfter, String timestampColumnName) { super(); this.seed = seed; this.size = size; this.deletedDocumentCount = deletedDocumentCount; this.refreshAfter = refreshAfter; // this.additionalAttributes = additionalAttributes; + this.timestampColumnName = timestampColumnName; } @Override @@ -461,6 +511,7 @@ public int hashCode() { result = prime * result + refreshAfter; result = prime * result + seed; result = prime * result + size; + result = prime * result + Objects.hashCode(timestampColumnName); return result; } @@ -488,6 +539,9 @@ public boolean equals(Object obj) { if (size != other.size) { return false; } + if (!Objects.equals(timestampColumnName, other.timestampColumnName)) { + return false; + } return true; } @@ -501,6 +555,7 @@ public static class Builder { private double deletedDocumentFraction = 0.06; private int refreshAfter = -1; private int segmentCount = 17; + private String timestampColumnName; public Builder() { super(); @@ -536,6 +591,11 @@ public Builder segmentCount(int segmentCount) { return this; } + public Builder timestampColumnName(String timestampColumnName) { + this.timestampColumnName = timestampColumnName; + return this; + } + public Key toKey() { if (deletedDocumentCount == -1) { this.deletedDocumentCount = (int) (this.size * deletedDocumentFraction); @@ -545,14 +605,14 @@ public Key toKey() { this.refreshAfter = this.size / this.segmentCount; } - return new Key(seed, size, deletedDocumentCount, refreshAfter); + return new Key(seed, size, deletedDocumentCount, refreshAfter, timestampColumnName); } public TestData get() { Key key = toKey(); try { - return cache.get(key, () -> new TestData(seed, size, deletedDocumentCount, refreshAfter)); + return cache.get(key, () -> new TestData(seed, size, deletedDocumentCount, refreshAfter, timestampColumnName)); } catch (ExecutionException e) { throw new RuntimeException(e); } @@ -641,6 +701,14 @@ public TestDocument get(String id) { public Set allIds() { return this.documents.keySet(); } + + public Map allDocs() { + ImmutableMap.Builder mapBuilder = ImmutableMap.builder(); + for (TestDocument testDocument : this.documents.values()) { + mapBuilder.put(testDocument.id, testDocument); + } + return mapBuilder.build(); + } } public static class TestDocument { @@ -756,22 +824,6 @@ public TestDocument withOnlyAttributes(String... attributes) { return new TestDocument(this.id, ImmutableMap.copyOf(newContent)); } - /* - public TestDocument withOnlyAttributes(Set attributes) { - Map newContent = new HashMap<>(); - for (String attri) - this.content.forEach((k, v) -> { - if (k.contains(".")) { - addAttributesRecursively(this.content, newContent, k.split("\\."), 0); - } else { - if (attributes.contains(k)) { - newContent.put(k, v); - } - } - }); - - } - */ public TestDocument applyTransform(DocumentTransformer transformerFunction) { return transformerFunction.transform(this); } diff --git a/src/integrationTest/java/org/opensearch/test/framework/data/TestDataStream.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestDataStream.java new file mode 100644 index 0000000000..6d73312b5e --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestDataStream.java @@ -0,0 +1,122 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.data; + +import java.util.Map; +import java.util.Set; + +import org.opensearch.action.admin.indices.datastream.CreateDataStreamAction; +import org.opensearch.action.admin.indices.datastream.DeleteDataStreamAction; +import org.opensearch.transport.client.Client; + +public class TestDataStream implements TestIndexOrAliasOrDatastream { + + private final String name; + private final TestData testData; + private final int rolloverAfter; + + public TestDataStream(String name, TestData testData, int rolloverAfter) { + this.name = name; + this.testData = testData; + this.rolloverAfter = rolloverAfter; + } + + @Override + public void create(Client client) { + client.admin().indices().createDataStream(new CreateDataStreamAction.Request(name)).actionGet(); + testData.putDocuments(client, name, rolloverAfter); + } + + @Override + public void delete(Client client) { + client.admin().indices().deleteDataStream(new DeleteDataStreamAction.Request(new String[] { name })).actionGet(); + } + + public String name() { + return name; + } + + public TestData testData() { + return testData; + } + + public static Builder name(String name) { + return new Builder().name(name); + } + + @Override + public String toString() { + return "Test data stream '" + name + '\''; + } + + public static class Builder { + private String name; + private final TestData.Builder testDataBuilder = new TestData.Builder().timestampColumnName("@timestamp") + .deletedDocumentFraction(0); + private TestData testData; + private int rolloverAfter = -1; + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder data(TestData data) { + this.testData = data; + return this; + } + + public Builder seed(int seed) { + testDataBuilder.seed(seed); + return this; + } + + public Builder documentCount(int size) { + testDataBuilder.documentCount(size); + return this; + } + + public Builder refreshAfter(int refreshAfter) { + testDataBuilder.refreshAfter(refreshAfter); + return this; + } + + public Builder rolloverAfter(int rolloverAfter) { + this.rolloverAfter = rolloverAfter; + return this; + } + + public Builder segmentCount(int segmentCount) { + testDataBuilder.segmentCount(segmentCount); + return this; + } + + public TestDataStream build() { + if (testData == null) { + testData = testDataBuilder.get(); + } + + return new TestDataStream(name, testData, rolloverAfter); + } + } + + @Override + public Set documentIds() { + return testData().getRetainedDocuments().keySet(); + } + + @Override + public Map documents() { + return testData().getRetainedDocuments(); + } + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/data/TestIndex.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndex.java new file mode 100644 index 0000000000..9fe0d49581 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndex.java @@ -0,0 +1,180 @@ +/* + * Copyright 2021-2022 floragunn GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.data; + +import java.util.Map; +import java.util.Set; + +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexNotFoundException; +import org.opensearch.transport.client.Client; + +public class TestIndex implements TestIndexOrAliasOrDatastream { + + private final String name; + private final Settings settings; + private final TestData testData; + + public TestIndex(String name, Settings settings, TestData testData) { + this.name = name; + this.settings = settings; + this.testData = testData; + } + + @Override + public void create(Client client) { + if (testData != null) { + testData.createIndex(client, name, settings); + } else { + client.admin().indices().create(new CreateIndexRequest(name).settings(settings)).actionGet(); + } + } + + @Override + public void delete(Client client) { + try { + client.admin().indices().delete(new DeleteIndexRequest(name)).actionGet(); + } catch (IndexNotFoundException e) { + // It is fine if the object to be deleted does not exist + } + } + + @Override + public String name() { + return name; + } + + @Override + public Set documentIds() { + return testData.documents().allIds(); + } + + @Override + public Map documents() { + return testData.documents().allDocs(); + } + + public TestData.TestDocument anyDocument() { + return testData.anyDocument(); + } + + public static Builder name(String name) { + return new Builder().name(name); + } + + public static class Builder { + private String name; + private Settings.Builder settings = Settings.builder(); + private TestData.Builder testDataBuilder = new TestData.Builder(); + private TestData testData; + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder setting(String name, int value) { + settings.put(name, value); + return this; + } + + public Builder shards(int value) { + settings.put("index.number_of_shards", value); + return this; + } + + public Builder hidden() { + settings.put("index.hidden", true); + return this; + } + + public Builder data(TestData testData) { + this.testData = testData; + return this; + } + + public Builder seed(int seed) { + testDataBuilder.seed(seed); + return this; + } + + public Builder documentCount(int size) { + testDataBuilder.documentCount(size); + return this; + } + + public TestIndex build() { + if (testData == null) { + testData = testDataBuilder.get(); + } + + return new TestIndex(name, settings.build(), testData); + } + + } + + /** + * This returns a magic TestIndexLike object symbolizing the internal OpenSearch security + * config index. This is supposed to be used with the IndexApiResponseMatchers. + */ + public static TestIndexOrAliasOrDatastream openSearchSecurityConfigIndex() { + return OPEN_SEARCH_SECURITY_CONFIG_INDEX; + } + + private final static TestIndexOrAliasOrDatastream OPEN_SEARCH_SECURITY_CONFIG_INDEX = new TestIndexOrAliasOrDatastream() { + + @Override + public String name() { + return ".opendistro_security"; + } + + @Override + public Map documents() { + return null; + } + + @Override + public void create(Client client) { + throw new UnsupportedOperationException(); + } + + @Override + public void delete(Client client) { + throw new UnsupportedOperationException(); + } + + @Override + public Set documentIds() { + return null; + } + }; + +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexOrAliasOrDatastream.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexOrAliasOrDatastream.java new file mode 100644 index 0000000000..2e1f6ae795 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexOrAliasOrDatastream.java @@ -0,0 +1,46 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.data; + +import java.util.Map; +import java.util.Set; + +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.transport.client.Client; + +public interface TestIndexOrAliasOrDatastream { + String name(); + + Set documentIds(); + + Map documents(); + + void create(Client client); + + void delete(Client client); + + static void createInitialTestObjects(LocalCluster cluster, TestIndexOrAliasOrDatastream... testIndexLikeArray) { + try (Client client = cluster.getInternalNodeClient()) { + for (TestIndexOrAliasOrDatastream testIndexLike : testIndexLikeArray) { + testIndexLike.create(client); + } + } + } + + static void delete(LocalCluster cluster, TestIndexOrAliasOrDatastream... testIndexLikeArray) { + try (Client client = cluster.getInternalNodeClient()) { + for (TestIndexOrAliasOrDatastream testIndexLike : testIndexLikeArray) { + testIndexLike.delete(client); + } + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexTemplate.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexTemplate.java new file mode 100644 index 0000000000..bd09a1b5e0 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestIndexTemplate.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.data; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import org.opensearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.opensearch.cluster.metadata.ComposableIndexTemplate; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.transport.client.Client; + +public class TestIndexTemplate { + public static final TestIndexTemplate DATA_STREAM_MINIMAL = new TestIndexTemplate("test_index_template_data_stream_minimal", "ds_*") + .dataStream() + .composedOf(TestComponentTemplate.DATA_STREAM_MINIMAL); + + private final String name; + private final ImmutableList indexPatterns; + private Object dataStream; + private ImmutableList composedOf = ImmutableList.of(); + private int priority = 0; + + public TestIndexTemplate(String name, String... indexPatterns) { + this.name = name; + this.indexPatterns = ImmutableList.copyOf(indexPatterns); + } + + public TestIndexTemplate dataStream() { + this.dataStream = ImmutableMap.of(); + return this; + } + + public TestIndexTemplate dataStream(String k, Object v) { + this.dataStream = ImmutableMap.of(k, v); + return this; + } + + public TestIndexTemplate composedOf(TestComponentTemplate... composedOf) { + this.composedOf = ImmutableList.copyOf(composedOf); + return this; + } + + public TestIndexTemplate priority(int priority) { + this.priority = priority; + return this; + } + + public String getName() { + return name; + } + + public List getComposedOf() { + return composedOf; + } + + public void create(Client client) throws Exception { + try (XContentBuilder builder = JsonXContent.contentBuilder().map(getAsMap())) { + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + BytesReference.bytes(builder).streamInput() + ) + ) { + client.admin() + .indices() + .execute( + PutComposableIndexTemplateAction.INSTANCE, + new PutComposableIndexTemplateAction.Request(name).indexTemplate(ComposableIndexTemplate.parse(parser)) + ) + .actionGet(); + } + } + } + + public Map getAsMap() { + return ImmutableMap.of( + "index_patterns", + indexPatterns, + "priority", + priority, + "data_stream", + dataStream, + "composed_of", + composedOf.stream().map(TestComponentTemplate::getName).collect(Collectors.toList()) + ); + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/data/TestMapping.java b/src/integrationTest/java/org/opensearch/test/framework/data/TestMapping.java new file mode 100644 index 0000000000..e323dc3733 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/data/TestMapping.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.data; + +import java.util.Map; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +public class TestMapping { + + private final ImmutableMap properties; + + public TestMapping(Property... properties) { + this.properties = ImmutableMap.copyOf( + ImmutableList.copyOf(properties).stream().collect(ImmutableMap.toImmutableMap(Property::getName, Property::getAsMap)) + ); + } + + public Map getAsMap() { + return ImmutableMap.of("properties", this.properties); + } + + public static class Property { + final String name; + final String type; + final String format; + + public Property(String name, String type, String format) { + this.name = name; + this.type = type; + this.format = format; + } + + public String getName() { + return name; + } + + public Map getAsMap() { + return ImmutableMap.of("type", type, "format", format); + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestDocumentMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestDocumentMatchers.java index e00f19455e..17e5f73dbb 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestDocumentMatchers.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestDocumentMatchers.java @@ -27,8 +27,8 @@ import org.hamcrest.DiagnosingMatcher; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.test.framework.TestData; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; +import org.opensearch.test.framework.data.TestData; /** * Matchers that can operate on responses of the OpenSearch REST APIs _search and _get; using various options like aggregations. diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java new file mode 100644 index 0000000000..ed1d9e8901 --- /dev/null +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestIndexMatchers.java @@ -0,0 +1,765 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.test.framework.matcher; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.hamcrest.Description; +import org.hamcrest.DiagnosingMatcher; +import org.hamcrest.Matcher; + +import org.opensearch.security.DefaultObjectMapper; +import org.opensearch.security.support.ConfigConstants; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.data.TestIndex; +import org.opensearch.test.framework.data.TestIndexOrAliasOrDatastream; + +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.spi.json.JacksonJsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; + +import static com.fasterxml.jackson.core.JsonToken.START_ARRAY; + +/** + * This class provides Hamcrest matchers that can be used as test oracles on the HTTP responses of index REST APIs. + *

+ * On a high level, the idea behind this class is like this: + *

    + *
  • Test users can be associated with IndexMatcher instances via the TestSecurityConfig.User.indexMatcher() method. These define the maximum index space the user can operate on. There may be several index matchers per user, targeting different groups of operations.
    • + *
  • The results of REST API calls can be also associated with a maximum space of indices the operation could work on. Combined with the user specific index matcher, one can determine the intersection of the allowed indices and thus the indices that are allowed in the particular case. The matchers support JSON path expressions to extract information on indices from the HTTP response bodies. See IndexAuthorizationReadOnlyIntTests for examples.
    • + *
+ */ +public class RestIndexMatchers { + + /** + * Matchers that are directly used on HTTP responses + */ + public interface OnResponseIndexMatcher extends IndexMatcher { + + /** + * Retrieves the actual indices from the HTTP response JSON body using this JSON path expression. + * If you are asserting on an HTTP response, specifying a JSON path is madatory. + */ + OnResponseIndexMatcher at(String jsonPath); + + /** + * Calculates the intersection of this index matcher and the given other index matcher. + * If this index matcher expects the indices a,b,c and the other index matcher expects b,c,d, + * the resulting matcher will expect b,c. + */ + OnResponseIndexMatcher reducedBy(IndexMatcher other); + + /** + * Asserts on a specific HTTP status code if the set of indices expected by this matcher is empty. + */ + OnResponseIndexMatcher whenEmpty(RestMatchers.HttpResponseMatcher statusCode); + + /** + * Checks whether the indices of this matcher are a subset of the other index matcher. + * If that is not the case, the given HTTP error will be expected in the response on which we are asserting. + */ + OnResponseIndexMatcher butFailIfIncomplete(IndexMatcher other, RestMatchers.HttpResponseMatcher statusCode); + + default IndexMatcher butForbiddenIfIncomplete(IndexMatcher other) { + return butFailIfIncomplete(other, RestMatchers.isForbidden()); + } + + /** + * Asserts that a TestRestClient.HttpResponse object refers exactly to a specific set of indices. + *

+ * Use this matcher like this: + * 

+         *     assertThat(httpResponse, containsExactly(index_a1, index_a2).at("hits.hits[*]._index"))
+         *
+ * This will verify that the HTTP response lists the indices index_a1 and index_a2 at the place specified by the JSON path query. + *

+ * It is possible to reduce the expected indices based on a test user this way: + * 

+         *     assertThat(httpResponse, containsExactly(index_a1, index_a2).at("hits.hits[*]._index").reducedBy(user.inderMatcher("search"))
+         *
+ * This will calculate the intersection of the indices specified here and of the indices specified with the user index matcher. + * The existence of exactly these indices will be asserted. + *

+ * This method has the special feature that you can also specify data streams; it will then assert that + * the backing indices of the data streams will be present in the result set. + */ + public static OnResponseIndexMatcher containsExactly(TestIndexOrAliasOrDatastream... testIndices) { + return containsExactly(Arrays.asList(testIndices)); + } + + public static OnResponseIndexMatcher containsExactly(Collection testIndices) { + Map indexNameMap = new HashMap<>(); + boolean containsOpenSearchSecurityIndex = false; + + for (TestIndexOrAliasOrDatastream testIndex : testIndices) { + if (testIndex == TestIndex.openSearchSecurityConfigIndex()) { + containsOpenSearchSecurityIndex = true; + } else { + indexNameMap.put(testIndex.name(), testIndex); + } + } + + return new ContainsExactlyMatcher(indexNameMap, containsOpenSearchSecurityIndex); + } + } + + /** + * Matchers that are associated with TestSecurityConfig.User objects via the indexMatcher() method + */ + public interface OnUserIndexMatcher extends IndexMatcher { + + static OnUserIndexMatcher limitedTo(TestIndexOrAliasOrDatastream... testIndices) { + return limitedTo(Arrays.asList(testIndices)); + } + + static OnUserIndexMatcher limitedTo(Collection testIndices) { + Map indexNameMap = new HashMap<>(); + + for (TestIndexOrAliasOrDatastream testIndex : testIndices) { + indexNameMap.put(testIndex.name(), testIndex); + } + + return new LimitedToMatcher(indexNameMap); + } + + static IndexMatcher unlimited() { + return new UnlimitedMatcher(); + } + + static IndexMatcher unlimitedIncludingOpenSearchSecurityIndex() { + return new UnlimitedMatcher(true); + } + + static IndexMatcher limitedToNone() { + return new LimitedToMatcher(Collections.emptyMap()); + } + + /** + * Adds the given indices to the set of indices this matcher is limited to. + * @param testIndices additional indices for the limitation. + * @return a new IndexMatcher instance with the new limit. + */ + OnUserIndexMatcher and(TestIndexOrAliasOrDatastream... testIndices); + } + + /** + * The returned IndexMatcher objects implement this interface. + */ + public interface IndexMatcher extends Matcher { + /** + * Checks whether this matcher expects an empty set of indices. + */ + boolean isEmpty(); + + /** + * Returns the number of indices expected by this matcher. + */ + int size(); + + /** + * Returns true if the set of indices is expected to contain the security config index + */ + boolean containsOpenSearchSecurityIndex(); + + /** + * Returns true if this matcher expects the given index to be present + */ + boolean covers(TestIndexOrAliasOrDatastream testIndex); + + /** + * Returns true if this matcher expects all the given indices to be present + */ + default boolean coversAll(TestIndexOrAliasOrDatastream... testIndices) { + return Stream.of(testIndices).allMatch(this::covers); + } + + default boolean coversAll(Collection testIndices) { + return testIndices.stream().allMatch(this::covers); + } + } + + // ---------------------------------------------------------------------------------- + // Actual matcher implementations + // (created by static methods in OnResponseIndexMatcher and OnUserIndexMatcher above) + // ---------------------------------------------------------------------------------- + + /** + * Base implementation for all matchers. The primary working mode of these matchers is to + * expect TestRestClient.HttpResponse objects and to extract index names from the response + * body via a jsonPath (specified with the at() method). However, the matchers will also + * work on any string collection; then, the json path is not necessary. + */ + static abstract class AbstractIndexMatcher extends DiagnosingMatcher implements IndexMatcher { + /** + * The indices expected by this matcher. + */ + protected final Map expectedIndices; + + /** + * The matcher will extract the indices from the REST response body using this JSON path expression. + */ + protected final String jsonPath; + + /** + * If the matcher expects an empty set of indices, this can actually mean two things: + *
    + *
  1. The response is expected to be successful (i.e. has a 200 OK status) and returns an empty set of indices
    2. + *
  2. The response has failed with a non 200 status code
    3. + *
+ * The expected status code is specified by this matcher. This matcher will be used to assert the status code when + * the expected set of indices is empty. + */ + protected final RestMatchers.HttpResponseMatcher statusCodeWhenEmpty; + + /** + * This is true if we also expect the .opendistro_security index. In case we gain further + * system indices that are present by default on an int test cluster, this can be expanded to cover also these. + */ + protected final boolean containsOpenSearchSecurityIndex; + + AbstractIndexMatcher(Map expectedIndices, boolean containsOpenSearchSecurityIndex) { + this.expectedIndices = expectedIndices; + this.jsonPath = null; + this.statusCodeWhenEmpty = RestMatchers.isOk(); + this.containsOpenSearchSecurityIndex = containsOpenSearchSecurityIndex; + } + + AbstractIndexMatcher( + Map expectedIndices, + boolean containsOpenSearchSecurityIndex, + String jsonPath, + RestMatchers.HttpResponseMatcher statusCodeWhenEmpty + ) { + this.expectedIndices = expectedIndices; + this.jsonPath = jsonPath; + this.statusCodeWhenEmpty = statusCodeWhenEmpty; + this.containsOpenSearchSecurityIndex = containsOpenSearchSecurityIndex; + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + TestRestClient.HttpResponse response = null; + + if (item instanceof TestRestClient.HttpResponse) { + response = (TestRestClient.HttpResponse) item; + + if (expectedIndices.isEmpty()) { + if (response.getStatusCode() != this.statusCodeWhenEmpty.statusCode()) { + mismatchDescription.appendText("Status was: ") + .appendValue(response.getStatusCode() + " " + response.getStatusReason()) + .appendText("\n\n") + .appendText(formatResponse(response)); + return false; + } + + if (response.getStatusCode() != 200) { + return true; + } + } + + try { + if (response.getBody().startsWith(START_ARRAY.asString())) { + item = DefaultObjectMapper.objectMapper.readValue(response.getBody(), List.class); + } else { + item = DefaultObjectMapper.objectMapper.readValue(response.getBody(), Map.class); + } + } catch (JsonProcessingException e) { + mismatchDescription.appendText("Unable to parse body: ").appendValue(e.getMessage()); + return false; + } + } + + if (jsonPath != null) { + Configuration config = Configuration.builder() + .jsonProvider(new JacksonJsonProvider()) + .mappingProvider(new JacksonMappingProvider()) + .evaluationListener() + .options(Option.SUPPRESS_EXCEPTIONS) + .build(); + + item = JsonPath.using(config).parse(item).read(jsonPath); + + if (item == null) { + mismatchDescription.appendText("Unable to find JSON Path: ") + .appendValue(jsonPath) + .appendText("\n\n") + .appendText(formatResponse(response)); + return false; + } + } + + if (!(item instanceof Collection)) { + item = Collections.singleton(item); + } + + return matchesImpl((Collection) item, mismatchDescription, response); + } + + /** + * This is called by the main matches() method after the indices have been extracted + * from the HTTP response body. The found indices will be passed as the actualItems parameter. + * + * @param actualItems The found indices. This is expected to be strings. + * @param mismatchDescription In case the matcher finds a mismatch, the description should be appended to this object. + * @param response The REST response we are asserting against. Optional. + * @return true if the assertion was successful, false it it failed. + */ + protected abstract boolean matchesImpl( + Collection actualItems, + Description mismatchDescription, + TestRestClient.HttpResponse response + ); + + @Override + public boolean isEmpty() { + return expectedIndices.isEmpty(); + } + + @Override + public int size() { + if (!containsOpenSearchSecurityIndex) { + return expectedIndices.size(); + } else { + return expectedIndices.size() + 1; + } + } + + @Override + public boolean containsOpenSearchSecurityIndex() { + return containsOpenSearchSecurityIndex; + } + + /** + * Calculates the intersection of the two given Map objects. + */ + protected Map testIndicesIntersection( + Map map1, + Map map2 + ) { + Map result = new HashMap<>(); + + for (Map.Entry entry : map1.entrySet()) { + String key = entry.getKey(); + TestIndexOrAliasOrDatastream index1 = entry.getValue(); + TestIndexOrAliasOrDatastream index2 = map2.get(key); + + if (index2 == null) { + continue; + } + + result.put(key, index1); + } + + return Collections.unmodifiableMap(result); + } + + protected ImmutableSet getExpectedIndices() { + return ImmutableSet.copyOf(expectedIndices.keySet()); + } + + /** + * Returns a formatted version of the response. This can be used in the mismatch description. + */ + protected static String formatResponse(TestRestClient.HttpResponse response) { + if (response == null) { + return ""; + } + + String start = response.getStatusCode() + " " + response.getStatusReason() + "\n"; + + if (response.isJsonContentType()) { + return start + response.bodyAsJsonNode().toPrettyString(); + } else { + return start + response.getBody(); + } + } + } + + /** + * This asserts that the item we assert on contains a set of indices that exactly corresponds to the expected + * indices (i.e., not fewer and not more indices). This is usually used to match against REST responses. + */ + static class ContainsExactlyMatcher extends AbstractIndexMatcher implements OnResponseIndexMatcher { + private static final Pattern DS_BACKING_INDEX_PATTERN = Pattern.compile("\\.ds-(.+)-[0-9]+"); + + ContainsExactlyMatcher(Map indexNameMap, boolean containsOpenSearchSecurityIndex) { + super(indexNameMap, containsOpenSearchSecurityIndex); + } + + ContainsExactlyMatcher( + Map indexNameMap, + boolean containsOpenSearchSecurityIndex, + String jsonPath, + RestMatchers.HttpResponseMatcher statusCodeWhenEmpty + ) { + super(indexNameMap, containsOpenSearchSecurityIndex, jsonPath, statusCodeWhenEmpty); + } + + @Override + public void describeTo(Description description) { + if (expectedIndices.isEmpty()) { + if (this.statusCodeWhenEmpty.statusCode() == 200) { + description.appendText("a 200 OK response with an empty result set"); + } else { + this.statusCodeWhenEmpty.describeTo(description); + description.appendText("a response with status code " + this.statusCodeWhenEmpty); + } + } else { + description.appendText( + "a 200 OK response with exactly the indices " + expectedIndices.keySet().stream().collect(Collectors.joining(", ")) + ); + } + } + + @Override + protected boolean matchesImpl(Collection actualItems, Description mismatchDescription, TestRestClient.HttpResponse response) { + // Flatten the collection + actualItems = actualItems.stream() + .flatMap(e -> e instanceof Collection ? ((Collection) e).stream() : Stream.of(e)) + .collect(Collectors.toSet()); + + return matchesByIndices(actualItems, mismatchDescription, response); + } + + protected boolean matchesByIndices( + Collection collection, + Description mismatchDescription, + TestRestClient.HttpResponse response + ) { + ImmutableSet expectedIndices = this.getExpectedIndices(); + ImmutableSet.Builder seenIndicesBuilder = ImmutableSet.builderWithExpectedSize(expectedIndices.size()); + ImmutableSet.Builder seenOpenSearchIndicesBuilder = new ImmutableSet.Builder<>(); + + for (Object object : collection) { + String index = object.toString(); + + if (containsOpenSearchSecurityIndex && (index.equals(ConfigConstants.OPENDISTRO_SECURITY_DEFAULT_CONFIG_INDEX))) { + seenOpenSearchIndicesBuilder.add(index); + } else if (index.startsWith(".ds-")) { + // We do a special treatment for data stream backing indices. We convert these to the normal data streams if expected + // indices contains these. + java.util.regex.Matcher matcher = DS_BACKING_INDEX_PATTERN.matcher(index); + + if (matcher.matches() && expectedIndices.contains(matcher.group(1))) { + seenIndicesBuilder.add(matcher.group(1)); + } else { + seenIndicesBuilder.add(index); + } + } else { + seenIndicesBuilder.add(index); + } + } + + ImmutableSet seenIndices = seenIndicesBuilder.build(); + + ImmutableSet unexpectedIndices = Sets.difference(seenIndices, expectedIndices).immutableCopy(); + ImmutableSet missingIndices = Sets.difference(expectedIndices, seenIndices).immutableCopy(); + + if (containsOpenSearchSecurityIndex && seenOpenSearchIndicesBuilder.build().size() == 0) { + missingIndices = ImmutableSet.builderWithExpectedSize(missingIndices.size() + 1) + .addAll(missingIndices) + .add(".opensearch indices") + .build(); + } + + if (unexpectedIndices.isEmpty() && missingIndices.isEmpty()) { + return true; + } else { + if (!missingIndices.isEmpty()) { + mismatchDescription.appendText("result does not contain expected indices; found: ") + .appendValue(seenIndices) + .appendText("; missing: ") + .appendValue(missingIndices) + .appendText("\n\n") + .appendText(formatResponse(response)); + } + + if (!unexpectedIndices.isEmpty()) { + mismatchDescription.appendText("result does contain indices that were not expected: ") + .appendValue(unexpectedIndices) + .appendText("\n\n") + .appendText(formatResponse(response)); + } + return false; + } + } + + @Override + public OnResponseIndexMatcher reducedBy(IndexMatcher other) { + if (other instanceof LimitedToMatcher) { + return new ContainsExactlyMatcher( + testIndicesIntersection(this.expectedIndices, ((LimitedToMatcher) other).expectedIndices), // + this.containsOpenSearchSecurityIndex && other.containsOpenSearchSecurityIndex(), // + this.jsonPath, + this.statusCodeWhenEmpty + ); + } else if (other instanceof ContainsExactlyMatcher) { + return new ContainsExactlyMatcher( + testIndicesIntersection(this.expectedIndices, ((ContainsExactlyMatcher) other).expectedIndices), // + this.containsOpenSearchSecurityIndex && other.containsOpenSearchSecurityIndex(), // + this.jsonPath, + this.statusCodeWhenEmpty + ); + } else if (other instanceof UnlimitedMatcher) { + return new ContainsExactlyMatcher( + this.expectedIndices, // + this.containsOpenSearchSecurityIndex && other.containsOpenSearchSecurityIndex(), // + this.jsonPath, + this.statusCodeWhenEmpty + ); + } else { + throw new RuntimeException("Unexpected argument " + other); + } + } + + @Override + public OnResponseIndexMatcher at(String jsonPath) { + return new ContainsExactlyMatcher(expectedIndices, containsOpenSearchSecurityIndex, jsonPath, statusCodeWhenEmpty); + } + + @Override + public OnResponseIndexMatcher whenEmpty(RestMatchers.HttpResponseMatcher statusCode) { + return new ContainsExactlyMatcher(expectedIndices, containsOpenSearchSecurityIndex, jsonPath, statusCode); + } + + @Override + public boolean covers(TestIndexOrAliasOrDatastream testIndex) { + return expectedIndices.containsKey(testIndex.name()); + } + + @Override + public OnResponseIndexMatcher butFailIfIncomplete(IndexMatcher other, RestMatchers.HttpResponseMatcher statusCode) { + if (other instanceof UnlimitedMatcher) { + return this; + } + + HashMap unmatched = new HashMap<>(this.expectedIndices); + unmatched.keySet().removeAll(((AbstractIndexMatcher) other).expectedIndices.keySet()); + + if (!unmatched.isEmpty()) { + return new StatusCodeMatcher(statusCode); + } else { + return this.reducedBy(other); + } + } + } + + /** + * Just asserts on the status code of a response. This is usually only used for failure status codes which + * are expected when the expected set of indices is empty. In this case, we do not apply any JSON path + * extractions, as we expect the response body to be just an error message. + */ + static class StatusCodeMatcher extends DiagnosingMatcher implements OnResponseIndexMatcher { + private RestMatchers.HttpResponseMatcher expectedStatusCode; + + public StatusCodeMatcher(RestMatchers.HttpResponseMatcher expectedStatusCode) { + this.expectedStatusCode = expectedStatusCode; + } + + @Override + public void describeTo(Description description) { + this.expectedStatusCode.describeTo(description); + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + return this.expectedStatusCode.matches(item, mismatchDescription); + } + + @Override + public boolean isEmpty() { + return true; + } + + @Override + public boolean containsOpenSearchSecurityIndex() { + return true; + } + + @Override + public int size() { + return 0; + } + + @Override + public boolean covers(TestIndexOrAliasOrDatastream testIndex) { + return false; + } + + @Override + public OnResponseIndexMatcher at(String jsonPath) { + return this; + } + + @Override + public OnResponseIndexMatcher reducedBy(IndexMatcher other) { + return this; + } + + @Override + public OnResponseIndexMatcher whenEmpty(RestMatchers.HttpResponseMatcher statusCode) { + return this; + } + + @Override + public OnResponseIndexMatcher butFailIfIncomplete(IndexMatcher other, RestMatchers.HttpResponseMatcher statusCode) { + return this; + } + } + + /** + * This asserts that the item we assert on contains not more than the expected indices. + * Usually, this is only associated with TestUser objects and used to reduce ContainsExactly matchers + * to even more limited ContainsExactly matchers. + */ + static class LimitedToMatcher extends AbstractIndexMatcher implements OnUserIndexMatcher { + + LimitedToMatcher(Map indexNameMap) { + super(indexNameMap, false); + } + + @Override + public void describeTo(Description description) { + if (expectedIndices.isEmpty()) { + if (this.statusCodeWhenEmpty.statusCode() == 200) { + description.appendText("a 200 OK response with an empty result set"); + } else { + this.statusCodeWhenEmpty.describeTo(description); + } + } else { + description.appendText( + "a 200 OK response no indices other than " + expectedIndices.keySet().stream().collect(Collectors.joining(", ")) + ); + } + } + + @Override + protected boolean matchesImpl(Collection actualItems, Description mismatchDescription, TestRestClient.HttpResponse response) { + return matchesByIndices(actualItems, mismatchDescription, response); + } + + @Override + public boolean covers(TestIndexOrAliasOrDatastream testIndex) { + return expectedIndices.containsKey(testIndex.name()); + } + + protected boolean matchesByIndices( + Collection collection, + Description mismatchDescription, + TestRestClient.HttpResponse response + ) { + ImmutableSet expectedIndices = this.getExpectedIndices(); + ImmutableSet.Builder seenIndicesBuilder = ImmutableSet.builderWithExpectedSize(expectedIndices.size()); + + for (Object object : collection) { + seenIndicesBuilder.add(object.toString()); + } + + ImmutableSet seenIndices = seenIndicesBuilder.build(); + ImmutableSet unexpectedIndices = Sets.difference(seenIndices, expectedIndices).immutableCopy(); + + if (unexpectedIndices.isEmpty()) { + return true; + } else { + mismatchDescription.appendText("result does contain indices that were not expected: ") + .appendValue(unexpectedIndices) + .appendText("\n\n") + .appendValue(formatResponse(response)); + return false; + } + } + + @Override + public OnUserIndexMatcher and(TestIndexOrAliasOrDatastream... testIndices) { + Map indexNameMap = new HashMap<>(this.expectedIndices); + + for (TestIndexOrAliasOrDatastream testIndex : testIndices) { + indexNameMap.put(testIndex.name(), testIndex); + } + + return new LimitedToMatcher(indexNameMap); + } + } + + /** + * This does no assertion on the expected indices. Usually, this is only associated with TestUser objects and used + * to signal that ContainsExactly matchers do not need to be reduced. + */ + static class UnlimitedMatcher extends DiagnosingMatcher implements OnUserIndexMatcher { + + private final boolean containsOpenSearchSecurityIndex; + + UnlimitedMatcher() { + this.containsOpenSearchSecurityIndex = false; + } + + UnlimitedMatcher(boolean containsOpenSearchSecurityIndex) { + this.containsOpenSearchSecurityIndex = containsOpenSearchSecurityIndex; + } + + @Override + public void describeTo(Description description) { + description.appendText("unlimited indices"); + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + if (item instanceof TestRestClient.HttpResponse response) { + if (response.getStatusCode() != 200) { + mismatchDescription.appendText("Expected status code 200 but status was: ") + .appendValue(response.getStatusCode() + " " + response.getStatusReason()); + return false; + } + } + + return true; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public boolean containsOpenSearchSecurityIndex() { + return containsOpenSearchSecurityIndex; + } + + @Override + public int size() { + throw new IllegalStateException("The UnlimitedMatcher cannot specify a size"); + } + + @Override + public boolean covers(TestIndexOrAliasOrDatastream testIndex) { + return true; + } + + @Override + public OnUserIndexMatcher and(TestIndexOrAliasOrDatastream... testIndices) { + return this; + } + } +} diff --git a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestMatchers.java b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestMatchers.java index 301f81b80e..96faab57c1 100644 --- a/src/integrationTest/java/org/opensearch/test/framework/matcher/RestMatchers.java +++ b/src/integrationTest/java/org/opensearch/test/framework/matcher/RestMatchers.java @@ -9,234 +9,172 @@ */ package org.opensearch.test.framework.matcher; +import java.util.Map; + +import com.google.common.collect.ImmutableMap; +import com.fasterxml.jackson.databind.JsonNode; import org.hamcrest.Description; import org.hamcrest.DiagnosingMatcher; -import org.opensearch.core.rest.RestStatus; import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; public class RestMatchers { private RestMatchers() {} - public static DiagnosingMatcher isOk() { - return new DiagnosingMatcher<>() { - - @Override - public void describeTo(Description description) { - description.appendText("Response has status 200 OK"); - } - - @Override - protected boolean matches(Object item, Description mismatchDescription) { - if (!(item instanceof HttpResponse)) { - mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); - return false; - } - - HttpResponse response = (HttpResponse) item; - - if (response.getStatusCode() == RestStatus.OK.getStatus()) { - return true; - } else { - mismatchDescription.appendText("Status is not 200 OK: ").appendValue(item); - return false; - } + public static HttpResponseMatcher isOk() { + return new HttpResponseMatcher(200, "OK"); + } - } + public static HttpResponseMatcher isCreated() { + return new HttpResponseMatcher(201, "Created"); + } - }; + public static OpenSearchErrorHttpResponseMatcher isForbidden() { + return new OpenSearchErrorHttpResponseMatcher(403, "Forbidden"); } public static DiagnosingMatcher isForbidden(String jsonPointer, String patternString) { - return new DiagnosingMatcher<>() { - - @Override - public void describeTo(Description description) { - description.appendText("Response has status 403 Forbidden with a JSON response that has the value ") - .appendValue(patternString) - .appendText(" at ") - .appendValue(jsonPointer); - } - - @Override - protected boolean matches(Object item, Description mismatchDescription) { - if (!(item instanceof HttpResponse)) { - mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); - return false; - } - - HttpResponse response = (HttpResponse) item; + return isForbidden().withAttribute(jsonPointer, patternString); + } - if (response.getStatusCode() != RestStatus.FORBIDDEN.getStatus()) { - mismatchDescription.appendText("Status is not 403 Forbidden: ").appendText("\n").appendValue(item); - return false; - } + public static OpenSearchErrorHttpResponseMatcher isBadRequest() { + return new OpenSearchErrorHttpResponseMatcher(400, "Bad Request"); + } - try { - String value = response.getTextFromJsonBody(jsonPointer); + public static DiagnosingMatcher isBadRequest(String jsonPointer, String patternString) { + return isBadRequest().withAttribute(jsonPointer, patternString); + } - if (value == null) { - mismatchDescription.appendText("Could not find value at " + jsonPointer).appendText("\n").appendValue(item); - return false; - } + public static OpenSearchErrorHttpResponseMatcher isNotImplemented() { + return new OpenSearchErrorHttpResponseMatcher(501, "Not Implemented"); + } - if (value.contains(patternString)) { - return true; - } else { - mismatchDescription.appendText("Value at " + jsonPointer + " does not match pattern: " + patternString + "\n") - .appendValue(item); - return false; - } - } catch (Exception e) { - mismatchDescription.appendText("Parsing request body failed with " + e).appendText("\n").appendValue(item); - return false; - } - } - }; + public static DiagnosingMatcher isMethodNotImplemented(String jsonPointer, String patternString) { + return isNotImplemented().withAttribute(jsonPointer, patternString); } - public static DiagnosingMatcher isBadRequest(String jsonPointer, String patternString) { - return new DiagnosingMatcher<>() { - - @Override - public void describeTo(Description description) { - description.appendText("Response has status 400 Bad Request with a JSON response that has the value ") - .appendValue(patternString) - .appendText(" at ") - .appendValue(jsonPointer); - } + public static OpenSearchErrorHttpResponseMatcher isInternalServerError() { + return new OpenSearchErrorHttpResponseMatcher(500, "Internal Server Error"); + } - @Override - protected boolean matches(Object item, Description mismatchDescription) { - if (!(item instanceof HttpResponse)) { - mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); - return false; - } + public static DiagnosingMatcher isInternalServerError(String jsonPointer, String patternString) { + return isInternalServerError().withAttribute(jsonPointer, patternString); + } - HttpResponse response = (HttpResponse) item; + public static OpenSearchErrorHttpResponseMatcher isNotFound() { + return new OpenSearchErrorHttpResponseMatcher(404, "Not Found"); + } - if (response.getStatusCode() != RestStatus.BAD_REQUEST.getStatus()) { - mismatchDescription.appendText("Status is not 400 Bad Request: ").appendText("\n").appendValue(item); - return false; - } + public static class HttpResponseMatcher extends DiagnosingMatcher { + final int statusCode; + final String statusName; + + HttpResponseMatcher(int statusCode, String statusName) { + this.statusCode = statusCode; + this.statusName = statusName; + } + + @Override + public void describeTo(Description description) { + description.appendText("Response has status " + statusCode + " " + statusName); + } + + @Override + protected boolean matches(Object item, Description mismatchDescription) { + if (!(item instanceof HttpResponse response)) { + mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); + return false; + } - try { - String value = response.getTextFromJsonBody(jsonPointer); + if (response.getStatusCode() == this.statusCode) { + return true; + } else { + mismatchDescription.appendText("Status is not " + statusCode + " " + statusName + ":\n").appendValue(item); + return false; + } + } - if (value == null) { - mismatchDescription.appendText("Could not find value at " + jsonPointer).appendText("\n").appendValue(item); - return false; - } + public int statusCode() { + return this.statusCode; + } - if (value.contains(patternString)) { - return true; - } else { - mismatchDescription.appendText("Value at " + jsonPointer + " does not match pattern: " + patternString + "\n") - .appendValue(item); - return false; - } - } catch (Exception e) { - mismatchDescription.appendText("Parsing request body failed with " + e).appendText("\n").appendValue(item); - return false; - } - } - }; } - public static DiagnosingMatcher isMethodNotImplemented(String jsonPointer, String patternString) { - return new DiagnosingMatcher<>() { - - @Override - public void describeTo(Description description) { - description.appendText("Response has status 501 Method Not Implemented with a JSON response that has the value ") - .appendValue(patternString) - .appendText(" at ") - .appendValue(jsonPointer); + public static class OpenSearchErrorHttpResponseMatcher extends HttpResponseMatcher { + final ImmutableMap attributes; + + OpenSearchErrorHttpResponseMatcher(int statusCode, String statusName) { + super(statusCode, statusName); + this.attributes = ImmutableMap.of(); + } + + OpenSearchErrorHttpResponseMatcher(int statusCode, String statusName, ImmutableMap attributes) { + super(statusCode, statusName); + this.attributes = attributes; + } + + public OpenSearchErrorHttpResponseMatcher withReason(String reason) { + return withAttribute("/error/reason", reason); + } + + public OpenSearchErrorHttpResponseMatcher withType(String type) { + return withAttribute("/error/type", type); + } + + public OpenSearchErrorHttpResponseMatcher withAttribute(String jsonPointer, String value) { + return new OpenSearchErrorHttpResponseMatcher( + this.statusCode, + this.statusName, + ImmutableMap.builder().putAll(this.attributes).put(jsonPointer, value).build() + ); + } + + @Override + public void describeTo(Description description) { + super.describeTo(description); + for (Map.Entry entry : this.attributes.entrySet()) { + description.appendText(" with " + entry.getKey() + " " + entry.getValue()); } + } - @Override - protected boolean matches(Object item, Description mismatchDescription) { - if (!(item instanceof HttpResponse)) { - mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); - return false; - } + @Override + protected boolean matches(Object item, Description mismatchDescription) { + if (!super.matches(item, mismatchDescription)) { + return false; + } - HttpResponse response = (HttpResponse) item; + HttpResponse response = (HttpResponse) item; + boolean result = true; - if (response.getStatusCode() != RestStatus.NOT_IMPLEMENTED.getStatus()) { - mismatchDescription.appendText("Status is not 501 Method Not Implemented: ").appendText("\n").appendValue(item); - return false; - } + if (!this.attributes.isEmpty()) { + JsonNode responseDocument; try { - String value = response.getTextFromJsonBody(jsonPointer); - - if (value == null) { - mismatchDescription.appendText("Could not find value at " + jsonPointer).appendText("\n").appendValue(item); - return false; - } - - if (value.contains(patternString)) { - return true; - } else { - mismatchDescription.appendText("Value at " + jsonPointer + " does not match pattern: " + patternString + "\n") - .appendValue(item); - return false; - } + responseDocument = response.bodyAsJsonNode(); } catch (Exception e) { mismatchDescription.appendText("Parsing request body failed with " + e).appendText("\n").appendValue(item); return false; } - } - }; - } - - public static DiagnosingMatcher isInternalServerError(String jsonPointer, String patternString) { - return new DiagnosingMatcher<>() { - - @Override - public void describeTo(Description description) { - description.appendText("Response has status 500 Internal Server Error with a JSON response that has the value ") - .appendValue(patternString) - .appendText(" at ") - .appendValue(jsonPointer); - } - @Override - protected boolean matches(Object item, Description mismatchDescription) { - if (!(item instanceof HttpResponse)) { - mismatchDescription.appendValue(item).appendText(" is not a HttpResponse"); - return false; - } - - HttpResponse response = (HttpResponse) item; - - if (response.getStatusCode() != RestStatus.INTERNAL_SERVER_ERROR.getStatus()) { - mismatchDescription.appendText("Status is not 500 Internal Server Error: ").appendText("\n").appendValue(item); - return false; + for (Map.Entry entry : this.attributes.entrySet()) { + String actualValue = responseDocument.at(entry.getKey()).asText(); + String expectedValue = entry.getValue(); + if (actualValue == null || !actualValue.contains(entry.getValue())) { + mismatchDescription.appendText(entry.getKey() + " is not " + expectedValue + ": ") + .appendValue(actualValue) + .appendText("\n"); + result = false; + } } + } - try { - String value = response.getTextFromJsonBody(jsonPointer); + if (!result) { + mismatchDescription.appendValue(item); + } - if (value == null) { - mismatchDescription.appendText("Could not find value at " + jsonPointer).appendText("\n").appendValue(item); - return false; - } + return result; + } - if (value.contains(patternString)) { - return true; - } else { - mismatchDescription.appendText("Value at " + jsonPointer + " does not match pattern: " + patternString + "\n") - .appendValue(item); - return false; - } - } catch (Exception e) { - mismatchDescription.appendText("Parsing request body failed with " + e).appendText("\n").appendValue(item); - return false; - } - } - }; } } diff --git a/src/integrationTest/resources/log4j2-test.properties b/src/integrationTest/resources/log4j2-test.properties index 18ef360ba2..d0bb23fa3f 100644 --- a/src/integrationTest/resources/log4j2-test.properties +++ b/src/integrationTest/resources/log4j2-test.properties @@ -53,6 +53,3 @@ logger.securenetty4transport.name = org.opensearch.transport.netty4.ssl.SecureNe logger.securenetty4transport.level = error logger.securenetty4transport.appenderRef.capturing.ref = logCapturingAppender - -logger.p.name=org.opensearch.security.privileges -logger.p.level=DEBUG