From 539324907406f79f876f234f96c7afe70d28f1d4 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Fri, 18 Apr 2025 11:48:45 -0400 Subject: [PATCH] Create sample plugin to demonstrate integrating with the security plugin --signoff --- build.gradle | 82 ++++-- sample-plugin/build.gradle | 271 ++++++++++++++++++ .../security/sample/SamplePluginTests.java | 57 ++++ .../security/sample/SamplePlugin.java | 18 ++ .../security/sample/ODFERestTestCase.java | 207 +++++++++++++ .../security/sample/SamplePluginIT.java | 43 +++ .../sample/SecureRestClientBuilder.java | 236 +++++++++++++++ .../security/sample/TrustStore.java | 71 +++++ settings.gradle | 3 + 9 files changed, 965 insertions(+), 23 deletions(-) create mode 100644 sample-plugin/build.gradle create mode 100644 sample-plugin/src/integrationTest/java/org/opensearch/security/sample/SamplePluginTests.java create mode 100644 sample-plugin/src/main/java/org/opensearch/security/sample/SamplePlugin.java create mode 100644 sample-plugin/src/test/java/org/opensearch/security/sample/ODFERestTestCase.java create mode 100644 sample-plugin/src/test/java/org/opensearch/security/sample/SamplePluginIT.java create mode 100644 sample-plugin/src/test/java/org/opensearch/security/sample/SecureRestClientBuilder.java create mode 100644 sample-plugin/src/test/java/org/opensearch/security/sample/TrustStore.java diff --git a/build.gradle b/build.gradle index 7bd1180b03..af4a2548b4 100644 --- a/build.gradle +++ b/build.gradle @@ -723,29 +723,29 @@ dependencies { compileOnly "org.opensearch:opensearch:${opensearch_version}" //integration test framework: - integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3') { - exclude(group: 'junit', module: 'junit') - } - integrationTestImplementation 'junit:junit:4.13.2' - integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" - integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" - integrationTestImplementation 'commons-io:commons-io:2.19.0' - integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" - integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}" - integrationTestImplementation 'org.hamcrest:hamcrest:2.2' - integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${bouncycastle_version}" - integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${bouncycastle_version}" - integrationTestImplementation('org.awaitility:awaitility:4.3.0') { - exclude(group: 'org.hamcrest', module: 'hamcrest') - } - integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14' - integrationTestImplementation "org.opensearch.plugin:mapper-size:${opensearch_version}" - integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.14" - integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.14" - integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.14" - integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16" - integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5" - integrationTestImplementation "org.mockito:mockito-core:5.17.0" +// integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3') { +// exclude(group: 'junit', module: 'junit') +// } +// integrationTestImplementation 'junit:junit:4.13.2' +// integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" +// integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" +// integrationTestImplementation 'commons-io:commons-io:2.19.0' +// integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" +// integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}" +// integrationTestImplementation 'org.hamcrest:hamcrest:2.2' +// integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${bouncycastle_version}" +// integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${bouncycastle_version}" +// integrationTestImplementation('org.awaitility:awaitility:4.3.0') { +// exclude(group: 'org.hamcrest', module: 'hamcrest') +// } +// integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14' +// integrationTestImplementation "org.opensearch.plugin:mapper-size:${opensearch_version}" +// integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.14" +// integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.14" +// integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.14" +// integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16" +// integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5" +// integrationTestImplementation "org.mockito:mockito-core:5.17.0" //spotless implementation('com.google.googlejavaformat:google-java-format:1.26.0') { @@ -753,6 +753,42 @@ dependencies { } } +allprojects { + configurations { + integrationTestImplementation.extendsFrom implementation + integrationTestRuntimeOnly.extendsFrom runtimeOnly + } + dependencies { + //integration test framework: + integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3') { + exclude(group: 'junit', module: 'junit') + } + integrationTestImplementation 'junit:junit:4.13.2' + integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" + integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}" + integrationTestImplementation 'commons-io:commons-io:2.19.0' + integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}" + integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}" + integrationTestImplementation 'org.hamcrest:hamcrest:2.2' + integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${bouncycastle_version}" + integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${bouncycastle_version}" + integrationTestImplementation('org.awaitility:awaitility:4.3.0') { + exclude(group: 'org.hamcrest', module: 'hamcrest') + } + integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14' + integrationTestImplementation "org.opensearch.plugin:mapper-size:${opensearch_version}" + integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.14" + integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.14" + integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.14" + integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16" + integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5" + integrationTestImplementation("org.mockito:mockito-core:5.16.1") { + exclude(group: 'net.bytebuddy', module: 'byte-buddy') + } + integrationTestImplementation "net.bytebuddy:byte-buddy:${versions.bytebuddy}" + } +} + jar { libsDirName = '.' into '', { diff --git a/sample-plugin/build.gradle b/sample-plugin/build.gradle new file mode 100644 index 0000000000..a6cd5bbfcd --- /dev/null +++ b/sample-plugin/build.gradle @@ -0,0 +1,271 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.testclusters' +apply plugin: 'opensearch.java-rest-test' + +import org.opensearch.gradle.test.RestIntegTestTask +import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask +import org.apache.tools.ant.taskdefs.condition.Os + +import java.util.concurrent.Callable + + +opensearchplugin { + name 'opensearch-security-sample-plugin' + description 'Sample plugin that demonstrates integrating with the security plugin' + classname 'org.opensearch.security.sample.SamplePlugin' +} + +ext { + projectSubstitutions = [:] + licenseFile = rootProject.file('LICENSE.txt') + noticeFile = rootProject.file('NOTICE.txt') + + ['esnode.pem', 'esnode-key.pem', 'kirk.pem', 'kirk-key.pem', 'root-ca.pem', 'sample.pem', 'test-kirk.jks'].forEach { file -> + File local = rootProject.file("bwc-test/src/test/resources/security/" + file) as File + processTestResources { + from(local) { + into 'security' + duplicatesStrategy = DuplicatesStrategy.INCLUDE + } + } + } +} + +repositories { + mavenLocal() + mavenCentral() + maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } +} + +//buildscript { +// ext { +// dependencies { +// classpath "org.opensearch.gradle:build-tools:${opensearch_version}" +// } +// } +//} + +configurations { + all { + resolutionStrategy { + force 'org.slf4j:slf4j-api:1.7.36' + force 'commons-codec:commons-codec:1.17.1' + force "org.apache.httpcomponents:httpclient:4.5.14" + force "org.apache.httpcomponents:httpcore:4.4.16" + force "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.1" + force "org.hamcrest:hamcrest:2.2" + force "com.fasterxml.jackson:jackson-bom:${versions.jackson}" + force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}" + force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + } + } +} + +// TODO Make this plugin have an integration test dependency on integrationTest from root project +dependencies { + // integrationTestImplementation project(":") + integrationTestImplementation rootProject.sourceSets.main.output + integrationTestImplementation rootProject.sourceSets.integrationTest.output + testImplementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + testImplementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + + integrationTestImplementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}" + integrationTestImplementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}" + integrationTestImplementation "com.google.guava:guava:${guava_version}" + integrationTestImplementation 'org.greenrobot:eventbus-java:3.3.1' + integrationTestImplementation 'org.ldaptive:ldaptive:1.2.3' + integrationTestImplementation 'com.password4j:password4j:1.8.2' + // Action privileges: check tables and compact collections + integrationTestImplementation 'com.selectivem.collections:special-collections-complete:1.4.0' + // JSON patch + integrationTestImplementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.16' + //Password generation + integrationTestImplementation 'org.passay:passay:1.6.5' + integrationTestImplementation 'org.slf4j:slf4j-api:1.7.36' + integrationTestImplementation "org.apache.commons:commons-lang3:${versions.commonslang}" + + integrationTestImplementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" + integrationTestImplementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + + integrationTestImplementation "org.opensearch:opensearch:${opensearch_version}" +} + +def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile +es_tmp_dir.mkdirs() + +File repo = file("$buildDir/testclusters/repo") +def _numNodes = findProperty('numNodes') as Integer ?: 1 + +licenseHeaders.enabled = true +validateNebulaPom.enabled = false +testingConventions.enabled = false +loggerUsageCheck.enabled = false + +javaRestTest.dependsOn(rootProject.assemble) +javaRestTest { + systemProperty 'tests.security.manager', 'false' +} +testClusters.javaRestTest { + testDistribution = 'INTEG_TEST' +} + +task integTest(type: RestIntegTestTask) { + description = "Run tests against a cluster" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath +} +tasks.named("check").configure { dependsOn(integTest) } + +//create source set 'integrationTest' +//add classes from the main source set to the compilation and runtime classpaths of the integrationTest +sourceSets { + integrationTest { + java { + srcDir file ('src/integrationTest/java') + compileClasspath += sourceSets.main.output + runtimeClasspath += sourceSets.main.output + } + resources { + srcDir file('src/integrationTest/resources') + } + processIntegrationTestResources { + duplicatesStrategy(DuplicatesStrategy.INCLUDE) + } + } +} + +task integrationTest(type: Test) { + testClassesDirs = sourceSets.integrationTest.output.classesDirs + classpath = sourceSets.integrationTest.runtimeClasspath +} + +configurations { + integrationTestImplementation.extendsFrom(rootProject.configurations.implementation) + integrationTestImplementation.extendsFrom(rootProject.configurations.testImplementation) + integrationTestImplementation.extendsFrom(rootProject.configurations.integrationTestImplementation) +} + +integTest { + if (project.hasProperty('excludeTests')) { + project.properties['excludeTests']?.replaceAll('\\s', '')?.split('[,;]')?.each { + exclude "${it}" + } + } + systemProperty 'tests.security.manager', 'false' + systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath + + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for + // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. + doFirst { + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can + // use longer timeouts for requests. + def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null + systemProperty 'cluster.debug', isDebuggingCluster + // Set number of nodes system property to be used in tests + systemProperty 'cluster.number_of_nodes', "${_numNodes}" + // There seems to be an issue when running multi node run or integ tasks with unicast_hosts + // not being written, the waitForAllConditions ensures it's written + getClusters().forEach { cluster -> + cluster.waitForAllConditions() + } + } + + // The -Dcluster.debug option makes the cluster debuggable; this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000' + } +} +project.getTasks().getByName('bundlePlugin').dependsOn(rootProject.tasks.getByName('build')) +Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin"); +Zip rootBundle = (Zip) rootProject.getTasks().getByName("bundlePlugin"); +integTest.dependsOn(bundle) +integTest.getClusters().forEach{c -> { + c.plugin(rootProject.getObjects().fileProperty().value(rootBundle.getArchiveFile())) + c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile())) +}} + +testClusters.integTest { + testDistribution = 'INTEG_TEST' + + // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 + if (_numNodes > 1) numberOfNodes = _numNodes + // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore + // i.e. we have to use a custom property to flag when we want to debug OpenSearch JVM + // since we also support multi node integration tests we increase debugPort per node + if (System.getProperty("cluster.debug") != null) { + def debugPort = 5005 + nodes.forEach { node -> + node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}") + debugPort += 1 + } + } + setting 'path.repo', repo.absolutePath +} + +afterEvaluate { + testClusters.integTest.nodes.each { node -> + def plugins = node.plugins + def firstPlugin = plugins.get(0) + if (firstPlugin.provider == project.bundlePlugin.archiveFile) { + plugins.remove(0) + plugins.add(firstPlugin) + } + + node.extraConfigFile("kirk.pem", rootProject.file("bwc-test/src/test/resources/security/kirk.pem")) + node.extraConfigFile("kirk-key.pem", rootProject.file("bwc-test/src/test/resources/security/kirk-key.pem")) + node.extraConfigFile("esnode.pem", rootProject.file("bwc-test/src/test/resources/security/esnode.pem")) + node.extraConfigFile("esnode-key.pem", rootProject.file("bwc-test/src/test/resources/security/esnode-key.pem")) + node.extraConfigFile("root-ca.pem", rootProject.file("bwc-test/src/test/resources/security/root-ca.pem")) + node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem") + node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem") + node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") + node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false") + node.setting("plugins.security.ssl.http.enabled", "true") + node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem") + node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem") + node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") + node.setting("plugins.security.allow_unsafe_democertificates", "true") + node.setting("plugins.security.allow_default_init_securityindex", "true") + node.setting("plugins.security.authcz.admin_dn", "\n - CN=kirk,OU=client,O=client,L=test,C=de") + node.setting("plugins.security.audit.type", "internal_opensearch") + node.setting("plugins.security.enable_snapshot_restore_privilege", "true") + node.setting("plugins.security.check_snapshot_restore_write_privileges", "true") + node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]") + } +} + +run { + doFirst { + // There seems to be an issue when running multi node run or integ tasks with unicast_hosts + // not being written, the waitForAllConditions ensures it's written + getClusters().forEach { cluster -> + cluster.waitForAllConditions() + } + } + useCluster testClusters.integTest +} + +// As of ES 7.7 the sample-extension-plugin is being added to the list of plugins for the testCluster during build before +// the security plugin is causing build failures. +// The security zip is added explicitly above but the sample-extension-plugin is added implicitly at some time during evaluation. +// Will need to do a deep dive to find out exactly what task adds the sample-extension-plugin and add security there but a temporary hack is to +// reorder the plugins list after evaluation but prior to task execution when the plugins are installed. +//afterEvaluate { +// testClusters.javaRestTest.nodes.each { node -> +// def nodePlugins = node.plugins +// def firstPlugin = nodePlugins.get(0) +// if (firstPlugin.provider == project.bundlePlugin.archiveFile) { +// nodePlugins.remove(0) +// nodePlugins.add(firstPlugin) +// } +// } +//} diff --git a/sample-plugin/src/integrationTest/java/org/opensearch/security/sample/SamplePluginTests.java b/sample-plugin/src/integrationTest/java/org/opensearch/security/sample/SamplePluginTests.java new file mode 100644 index 0000000000..b88b0f5f30 --- /dev/null +++ b/sample-plugin/src/integrationTest/java/org/opensearch/security/sample/SamplePluginTests.java @@ -0,0 +1,57 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.security.sample; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import org.apache.http.HttpStatus; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.opensearch.test.framework.cluster.ClusterManager; +import org.opensearch.test.framework.cluster.LocalCluster; +import org.opensearch.test.framework.cluster.TestRestClient; +import org.opensearch.test.framework.cluster.TestRestClient.HttpResponse; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.test.framework.TestSecurityConfig.AuthcDomain.AUTHC_HTTPBASIC_INTERNAL; +import static org.opensearch.test.framework.TestSecurityConfig.User.USER_ADMIN; + +@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) +@ThreadLeakScope(ThreadLeakScope.Scope.NONE) +public class SamplePluginTests { + + @ClassRule + public static LocalCluster cluster = new LocalCluster.Builder().clusterManager(ClusterManager.SINGLENODE) + .plugin(SamplePlugin.class) + .anonymousAuth(true) + .authc(AUTHC_HTTPBASIC_INTERNAL) + .users(USER_ADMIN) + .build(); + + @Test + public void testSecurityRoles() throws Exception { + try (TestRestClient client = cluster.getRestClient(USER_ADMIN)) { + HttpResponse response = client.getAuthInfo(); + response.assertStatusCode(HttpStatus.SC_OK); + + // Check username + assertThat(response.getTextFromJsonBody("/user_name"), equalTo("admin")); + HttpResponse pluginsResponse = client.get("_cat/plugins?s=component&h=name,component,version,description"); + assertThat(pluginsResponse.getBody(), containsString("org.opensearch.security.OpenSearchSecurityPlugin")); + assertThat(pluginsResponse.getBody(), containsString("org.opensearch.security.sample.SamplePlugin")); + } + } + +} diff --git a/sample-plugin/src/main/java/org/opensearch/security/sample/SamplePlugin.java b/sample-plugin/src/main/java/org/opensearch/security/sample/SamplePlugin.java new file mode 100644 index 0000000000..da18f89262 --- /dev/null +++ b/sample-plugin/src/main/java/org/opensearch/security/sample/SamplePlugin.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.security.sample; + +import org.opensearch.plugins.Plugin; + +/** + * Sample Plugin used to demonstrate integrating with the security plugin + */ +public class SamplePlugin extends Plugin { + +} diff --git a/sample-plugin/src/test/java/org/opensearch/security/sample/ODFERestTestCase.java b/sample-plugin/src/test/java/org/opensearch/security/sample/ODFERestTestCase.java new file mode 100644 index 0000000000..eb7f4ba3b9 --- /dev/null +++ b/sample-plugin/src/test/java/org/opensearch/security/sample/ODFERestTestCase.java @@ -0,0 +1,207 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.security.sample; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import javax.net.ssl.SSLEngine; + +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.function.Factory; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.reactor.ssl.TlsDetails; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.util.Timeout; +import org.junit.After; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.client.WarningFailureException; +import org.opensearch.common.io.PathUtils; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.util.concurrent.ThreadContext; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.MediaType; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +public abstract class ODFERestTestCase extends OpenSearchRestTestCase { + + protected boolean isHttps() { + boolean isHttps = Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); + if (isHttps) { + // currently only external cluster is supported for security enabled testing + if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) { + throw new RuntimeException("cluster url should be provided for security enabled testing"); + } + } + + return isHttps; + } + + @Override + protected String getProtocol() { + return isHttps() ? "https" : "http"; + } + + @Override + protected Settings restAdminSettings() { + return Settings.builder() + .put("http.port", 9200) + .put("plugins.security.ssl.http.enabled", isHttps()) + .put("plugins.security.ssl.http.pemcert_filepath", "sample.pem") + .put("plugins.security.ssl.http.keystore_filepath", "test-kirk.jks") + .put("plugins.security.ssl.http.keystore_password", "changeit") + .build(); + // return Settings.builder().put("strictDeprecationMode", false).put("http.port", 9200).build(); + } + + @Override + protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { + boolean strictDeprecationMode = settings.getAsBoolean("strictDeprecationMode", true); + RestClientBuilder builder = RestClient.builder(hosts); + if (isHttps()) { + String keystore = settings.get("plugins.security.ssl.http.keystore_filepath"); + if (Objects.nonNull(keystore)) { + URI uri = null; + try { + uri = this.getClass().getClassLoader().getResource("security/sample.pem").toURI(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + Path configPath = PathUtils.get(uri).getParent().toAbsolutePath(); + return new SecureRestClientBuilder(settings, configPath, hosts).build(); + } else { + configureHttpsClient(builder, settings); + builder.setStrictDeprecationMode(strictDeprecationMode); + return builder.build(); + } + } else { + configureClient(builder, settings); + builder.setStrictDeprecationMode(strictDeprecationMode); + return builder.build(); + } + + } + + @SuppressWarnings("unchecked") + @After + protected void wipeAllODFEIndices() throws IOException { + Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); + MediaType mediaType = MediaType.fromMediaType(response.getEntity().getContentType()); + try ( + XContentParser parser = mediaType.xContent() + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + response.getEntity().getContent() + ) + ) { + XContentParser.Token token = parser.nextToken(); + List> parserList = null; + if (token == XContentParser.Token.START_ARRAY) { + parserList = parser.listOrderedMap().stream().map(obj -> (Map) obj).collect(Collectors.toList()); + } else { + parserList = Collections.singletonList(parser.mapOrdered()); + } + + for (Map index : parserList) { + String indexName = (String) index.get("index"); + if (indexName != null && !".opendistro_security".equals(indexName)) { + try { + adminClient().performRequest(new Request("DELETE", "/" + indexName)); + } catch (WarningFailureException ignore) {} + } + } + } + } + + protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { + Map headers = new HashMap<>(ThreadContext.buildDefaultHeaders(settings)); + String userName = Optional.ofNullable(System.getProperty("user")).orElseThrow(() -> new RuntimeException("user name is missing")); + String password = Optional.ofNullable(System.getProperty("password")) + .orElseThrow(() -> new RuntimeException("password is missing")); + headers.put( + "Authorization", + "Basic " + Base64.getEncoder().encodeToString((userName + ":" + password).getBytes(StandardCharsets.UTF_8)) + ); + headers.put("Content-Type", "application/json"); + Header[] defaultHeaders = new Header[headers.size()]; + int i = 0; + for (Map.Entry entry : headers.entrySet()) { + defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue()); + } + builder.setDefaultHeaders(defaultHeaders); + builder.setHttpClientConfigCallback(httpClientBuilder -> { + try { + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) + // disable the certificate since our testing cluster just uses the default security configuration + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + // See please https://issues.apache.org/jira/browse/HTTPCLIENT-2219 + .setTlsDetailsFactory(new Factory() { + @Override + public TlsDetails create(final SSLEngine sslEngine) { + return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); + } + }) + .build(); + + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build(); + + return httpClientBuilder.setConnectionManager(connectionManager); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT); + final TimeValue socketTimeout = TimeValue.parseTimeValue( + socketTimeoutString == null ? "60s" : socketTimeoutString, + CLIENT_SOCKET_TIMEOUT + ); + builder.setRequestConfigCallback( + conf -> conf.setResponseTimeout(Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis()))) + ); + if (settings.hasValue(CLIENT_PATH_PREFIX)) { + builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); + } + } + + /** + * wipeAllIndices won't work since it cannot delete security index. Use wipeAllODFEIndices instead. + */ + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } +} diff --git a/sample-plugin/src/test/java/org/opensearch/security/sample/SamplePluginIT.java b/sample-plugin/src/test/java/org/opensearch/security/sample/SamplePluginIT.java new file mode 100644 index 0000000000..1d5477e901 --- /dev/null +++ b/sample-plugin/src/test/java/org/opensearch/security/sample/SamplePluginIT.java @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.security.sample; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.junit.Assert; + +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.NamedXContentRegistry; + +public class SamplePluginIT extends ODFERestTestCase { + + @SuppressWarnings("unchecked") + public void testPluginsAreInstalled() throws IOException { + Request request = new Request("GET", "/_cat/plugins?s=component&h=name,component,version,description&format=json"); + Response response = client().performRequest(request); + List pluginsList = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + LoggingDeprecationHandler.INSTANCE, + response.getEntity().getContent() + ).list(); + Assert.assertTrue( + pluginsList.stream().map(o -> (Map) o).anyMatch(plugin -> plugin.get("component").equals("opensearch-security")) + ); + Assert.assertTrue( + pluginsList.stream() + .map(o -> (Map) o) + .anyMatch(plugin -> plugin.get("component").equals("opensearch-security-sample-plugin")) + ); + } +} diff --git a/sample-plugin/src/test/java/org/opensearch/security/sample/SecureRestClientBuilder.java b/sample-plugin/src/test/java/org/opensearch/security/sample/SecureRestClientBuilder.java new file mode 100644 index 0000000000..3ad342dc65 --- /dev/null +++ b/sample-plugin/src/test/java/org/opensearch/security/sample/SecureRestClientBuilder.java @@ -0,0 +1,236 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.security.sample; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.util.ArrayList; +import java.util.Arrays; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; + +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.CredentialsProvider; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.TrustSelfSignedStrategy; +import org.apache.hc.core5.function.Factory; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.reactor.ssl.TlsDetails; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.util.Timeout; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.opensearch.OpenSearchException; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; + +/** + * Provides builder to create low-level and high-level REST client to make calls to OpenSearch. + * + * Sample usage: + * SecureRestClientBuilder builder = new SecureRestClientBuilder(settings).build() + * RestClient restClient = builder.build(); + * + * Other usage: + * RestClient restClient = new SecureRestClientBuilder("localhost", 9200, false) + * .setUserPassword("admin", "myStrongPassword123") + * .setTrustCerts(trustStorePath) + * .build(); + * + * + * If https is enabled, creates RestClientBuilder using self-signed certificates or passed pem + * as trusted. + * + * If https is not enabled, creates a http based client. + */ +public class SecureRestClientBuilder { + + private final boolean httpSSLEnabled; + private final String user; + private final String passwd; + private final ArrayList hosts = new ArrayList<>(); + + private final Path configPath; + private final Settings settings; + + private int defaultConnectTimeOutMSecs = 5000; + private int defaultSoTimeoutMSecs = 10000; + private int defaultConnRequestTimeoutMSecs = 3 * 60 * 1000; /* 3 mins */ + private int defaultMaxConnPerRoute = RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; + private int defaultMaxConnTotal = RestClientBuilder.DEFAULT_MAX_CONN_TOTAL; + + private static final Logger log = LogManager.getLogger(SecureRestClientBuilder.class); + + public SecureRestClientBuilder(Settings settings, Path configPath, HttpHost[] httpHosts) { + this.httpSSLEnabled = settings.getAsBoolean("plugins.security.ssl.http.enabled", false); + this.settings = settings; + this.configPath = configPath; + this.user = null; + this.passwd = null; + hosts.addAll(Arrays.asList(httpHosts)); + } + + /** + * Creates a low-level Rest client. + * @return + * @throws IOException + */ + public RestClient build() throws IOException { + return createRestClientBuilder().build(); + } + + private RestClientBuilder createRestClientBuilder() throws IOException { + RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[hosts.size()])); + + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + return requestConfigBuilder.setConnectTimeout(Timeout.ofMilliseconds(defaultConnectTimeOutMSecs)) + .setResponseTimeout(Timeout.ofMilliseconds(defaultSoTimeoutMSecs)) + .setConnectionRequestTimeout(Timeout.ofMilliseconds(defaultConnRequestTimeoutMSecs)); + } + }); + + final SSLContext sslContext; + try { + sslContext = createSSLContext(); + } catch (GeneralSecurityException | IOException ex) { + throw new IOException(ex); + } + final CredentialsProvider credentialsProvider = createCredsProvider(); + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { + if (sslContext != null) { + TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setSslContext(sslContext) + // See please https://issues.apache.org/jira/browse/HTTPCLIENT-2219 + .setTlsDetailsFactory(new Factory() { + @Override + public TlsDetails create(final SSLEngine sslEngine) { + return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); + } + }) + .build(); + PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .setMaxConnPerRoute(defaultMaxConnPerRoute) + .setMaxConnTotal(defaultMaxConnTotal) + .build(); + httpClientBuilder.setConnectionManager(connectionManager); + } + if (credentialsProvider != null) { + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + } + return httpClientBuilder; + } + }); + return builder; + } + + private SSLContext createSSLContext() throws IOException, GeneralSecurityException { + SSLContextBuilder builder = new SSLContextBuilder(); + if (httpSSLEnabled) { + // Handle trust store + String pemFile = getTrustPem(); + if (Strings.isNullOrEmpty(pemFile)) { + builder.loadTrustMaterial(null, new TrustSelfSignedStrategy()); + } else { + String pem = resolve(pemFile, configPath); + KeyStore trustStore = new TrustStore(pem).create(); + builder.loadTrustMaterial(trustStore, null); + } + + // Handle key store. + KeyStore keyStore = getKeyStore(); + if (keyStore != null) { + builder.loadKeyMaterial(keyStore, getKeystorePasswd().toCharArray()); + } + + } + return builder.build(); + } + + private CredentialsProvider createCredsProvider() { + if (Strings.isNullOrEmpty(user) || Strings.isNullOrEmpty(passwd)) return null; + + final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(new AuthScope(null, -1), new UsernamePasswordCredentials(user, passwd.toCharArray())); + return credentialsProvider; + } + + private String resolve(final String originalFile, final Path configPath) { + String path = null; + if (originalFile != null && originalFile.length() > 0) { + path = configPath.resolve(originalFile).toAbsolutePath().toString(); + log.debug("Resolved {} to {} against {}", originalFile, path, configPath.toAbsolutePath().toString()); + } + + if (path == null || path.length() == 0) { + throw new OpenSearchException("Empty file path for " + originalFile); + } + + if (Files.isDirectory(Paths.get(path), LinkOption.NOFOLLOW_LINKS)) { + throw new OpenSearchException("Is a directory: " + path + " Expected a file for " + originalFile); + } + + if (!Files.isReadable(Paths.get(path))) { + throw new OpenSearchException( + "Unable to read " + + path + + " (" + + Paths.get(path) + + "). Please make sure this files exists and is readable regarding to permissions. Property: " + + originalFile + ); + } + if ("".equals(path)) { + path = null; + } + return path; + } + + private String getTrustPem() { + return settings.get("plugins.security.ssl.http.pemcert_filepath", null); + } + + private String getKeystorePasswd() { + return settings.get("plugins.security.ssl.http.keystore_password", null); + } + + private KeyStore getKeyStore() throws IOException, GeneralSecurityException { + KeyStore keyStore = KeyStore.getInstance("jks"); + String keyStoreFile = settings.get("plugins.security.ssl.http.keystore_filepath", null); + String passwd = settings.get("plugins.security.ssl.http.keystore_password", null); + if (Strings.isNullOrEmpty(keyStoreFile) || Strings.isNullOrEmpty(passwd)) { + return null; + } + String keyStorePath = resolve(keyStoreFile, configPath); + try (InputStream is = Files.newInputStream(Paths.get(keyStorePath))) { + keyStore.load(is, passwd.toCharArray()); + } + return keyStore; + } +} diff --git a/sample-plugin/src/test/java/org/opensearch/security/sample/TrustStore.java b/sample-plugin/src/test/java/org/opensearch/security/sample/TrustStore.java new file mode 100644 index 0000000000..ac9c31c661 --- /dev/null +++ b/sample-plugin/src/test/java/org/opensearch/security/sample/TrustStore.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.security.sample; + +import java.io.FileInputStream; +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.util.Collection; + +/** + * Helper class to read raw pem files to keystore. + */ +public class TrustStore { + + private final String effectiveKeyAlias = "al"; + private final String storeType = "JKS"; + private final String certType = "X.509"; + private final String cert; + + public TrustStore(final String file) { + cert = file; + } + + public KeyStore create() throws IOException, GeneralSecurityException { + X509Certificate[] trustCerts = loadCertificatesFromFile(cert); + return toTrustStore(effectiveKeyAlias, trustCerts); + } + + private X509Certificate[] loadCertificatesFromFile(String file) throws IOException, GeneralSecurityException { + if (file == null) { + return null; + } + CertificateFactory fact = CertificateFactory.getInstance(certType); + try (FileInputStream is = new FileInputStream(file)) { + Collection certs = fact.generateCertificates(is); + X509Certificate[] x509Certs = new X509Certificate[certs.size()]; + int i = 0; + for (Certificate cert : certs) { + x509Certs[i++] = (X509Certificate) cert; + } + return x509Certs; + } + } + + private KeyStore toTrustStore(final String trustCertificatesAliasPrefix, final X509Certificate[] trustCertificates) throws IOException, + GeneralSecurityException { + if (trustCertificates == null) { + return null; + } + KeyStore ks = KeyStore.getInstance(storeType); + ks.load(null); + + if (trustCertificates != null) { + for (int i = 0; i < trustCertificates.length; i++) { + X509Certificate x509Certificate = trustCertificates[i]; + ks.setCertificateEntry(trustCertificatesAliasPrefix + "_" + i, x509Certificate); + } + } + return ks; + } +} diff --git a/settings.gradle b/settings.gradle index 1c3e7ff5aa..bc39c4ab02 100644 --- a/settings.gradle +++ b/settings.gradle @@ -5,3 +5,6 @@ */ rootProject.name = 'opensearch-security' + +include "sample-plugin" +project(":sample-plugin").name = rootProject.name + "-sample-plugin"