Create sample plugin to demonstrate integrating with the security plugin

build.gradle

@@ -723,36 +723,72 @@ dependencies {
     compileOnly "org.opensearch:opensearch:${opensearch_version}"
 
     //integration test framework:
-    integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3') {
-        exclude(group: 'junit', module: 'junit')
-    }
-    integrationTestImplementation 'junit:junit:4.13.2'
-    integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
-    integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
-    integrationTestImplementation 'commons-io:commons-io:2.19.0'
-    integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
-    integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
-    integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
-    integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${bouncycastle_version}"
-    integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${bouncycastle_version}"
-    integrationTestImplementation('org.awaitility:awaitility:4.3.0') {
-        exclude(group: 'org.hamcrest', module: 'hamcrest')
-    }
-    integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'
-    integrationTestImplementation "org.opensearch.plugin:mapper-size:${opensearch_version}"
-    integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.14"
-    integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.14"
-    integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.14"
-    integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16"
-    integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
-    integrationTestImplementation "org.mockito:mockito-core:5.17.0"
+//    integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3') {
+//        exclude(group: 'junit', module: 'junit')
+//    }
+//    integrationTestImplementation 'junit:junit:4.13.2'
+//    integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
+//    integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
+//    integrationTestImplementation 'commons-io:commons-io:2.19.0'
+//    integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
+//    integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
+//    integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
+//    integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${bouncycastle_version}"
+//    integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${bouncycastle_version}"
+//    integrationTestImplementation('org.awaitility:awaitility:4.3.0') {
+//        exclude(group: 'org.hamcrest', module: 'hamcrest')
+//    }
+//    integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'
+//    integrationTestImplementation "org.opensearch.plugin:mapper-size:${opensearch_version}"
+//    integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.14"
+//    integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.14"
+//    integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.14"
+//    integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16"
+//    integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
+//    integrationTestImplementation "org.mockito:mockito-core:5.17.0"
 
     //spotless
     implementation('com.google.googlejavaformat:google-java-format:1.26.0') {
         exclude group: 'com.google.guava'
     }
 }
 
+allprojects {
+    configurations {
+        integrationTestImplementation.extendsFrom implementation
+        integrationTestRuntimeOnly.extendsFrom runtimeOnly
+    }
+    dependencies {
+        //integration test framework:
+        integrationTestImplementation('com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3') {
+            exclude(group: 'junit', module: 'junit')
+        }
+        integrationTestImplementation 'junit:junit:4.13.2'
+        integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
+        integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
+        integrationTestImplementation 'commons-io:commons-io:2.19.0'
+        integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
+        integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
+        integrationTestImplementation 'org.hamcrest:hamcrest:2.2'
+        integrationTestImplementation "org.bouncycastle:bcpkix-jdk18on:${bouncycastle_version}"
+        integrationTestImplementation "org.bouncycastle:bcutil-jdk18on:${bouncycastle_version}"
+        integrationTestImplementation('org.awaitility:awaitility:4.3.0') {
+            exclude(group: 'org.hamcrest', module: 'hamcrest')
+        }
+        integrationTestImplementation 'com.unboundid:unboundid-ldapsdk:4.0.14'
+        integrationTestImplementation "org.opensearch.plugin:mapper-size:${opensearch_version}"
+        integrationTestImplementation "org.apache.httpcomponents:httpclient-cache:4.5.14"
+        integrationTestImplementation "org.apache.httpcomponents:httpclient:4.5.14"
+        integrationTestImplementation "org.apache.httpcomponents:fluent-hc:4.5.14"
+        integrationTestImplementation "org.apache.httpcomponents:httpcore:4.4.16"
+        integrationTestImplementation "org.apache.httpcomponents:httpasyncclient:4.1.5"
+        integrationTestImplementation("org.mockito:mockito-core:5.16.1") {
+            exclude(group: 'net.bytebuddy', module: 'byte-buddy')
+        }
+        integrationTestImplementation "net.bytebuddy:byte-buddy:${versions.bytebuddy}"
+    }
+}
+
 jar {
     libsDirName = '.'
     into '', {

sample-plugin/build.gradle

@@ -0,0 +1,271 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+apply plugin: 'opensearch.opensearchplugin'
+apply plugin: 'opensearch.testclusters'
+apply plugin: 'opensearch.java-rest-test'
+
+import org.opensearch.gradle.test.RestIntegTestTask
+import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
+import org.apache.tools.ant.taskdefs.condition.Os
+
+import java.util.concurrent.Callable
+
+
+opensearchplugin {
+    name 'opensearch-security-sample-plugin'
+    description 'Sample plugin that demonstrates integrating with the security plugin'
+    classname 'org.opensearch.security.sample.SamplePlugin'
+}
+
+ext {
+    projectSubstitutions = [:]
+    licenseFile = rootProject.file('LICENSE.txt')
+    noticeFile = rootProject.file('NOTICE.txt')
+
+    ['esnode.pem', 'esnode-key.pem', 'kirk.pem', 'kirk-key.pem', 'root-ca.pem', 'sample.pem', 'test-kirk.jks'].forEach { file ->
+        File local = rootProject.file("bwc-test/src/test/resources/security/" + file) as File
+        processTestResources {
+            from(local) {
+                into 'security'
+                duplicatesStrategy = DuplicatesStrategy.INCLUDE
+            }
+        }
+    }
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
+}
+
+//buildscript {
+//    ext {
+//        dependencies {
+//            classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
+//        }
+//    }
+//}
+
+configurations {
+    all {
+        resolutionStrategy {
+            force 'org.slf4j:slf4j-api:1.7.36'
+            force 'commons-codec:commons-codec:1.17.1'
+            force "org.apache.httpcomponents:httpclient:4.5.14"
+            force "org.apache.httpcomponents:httpcore:4.4.16"
+            force "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.1"
+            force "org.hamcrest:hamcrest:2.2"
+            force "com.fasterxml.jackson:jackson-bom:${versions.jackson}"
+            force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+            force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}"
+            force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+        }
+    }
+}
+
+// TODO Make this plugin have an integration test dependency on integrationTest from root project
+dependencies {
+    // integrationTestImplementation project(":")
+    integrationTestImplementation rootProject.sourceSets.main.output
+    integrationTestImplementation rootProject.sourceSets.integrationTest.output
+    testImplementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+    testImplementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+
+    integrationTestImplementation "org.opensearch.plugin:transport-netty4-client:${opensearch_version}"
+    integrationTestImplementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}"
+    integrationTestImplementation "com.google.guava:guava:${guava_version}"
+    integrationTestImplementation 'org.greenrobot:eventbus-java:3.3.1'
+    integrationTestImplementation 'org.ldaptive:ldaptive:1.2.3'
+    integrationTestImplementation 'com.password4j:password4j:1.8.2'
+    // Action privileges: check tables and compact collections
+    integrationTestImplementation 'com.selectivem.collections:special-collections-complete:1.4.0'
+    // JSON patch
+    integrationTestImplementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.16'
+    //Password generation
+    integrationTestImplementation 'org.passay:passay:1.6.5'
+    integrationTestImplementation 'org.slf4j:slf4j-api:1.7.36'
+    integrationTestImplementation "org.apache.commons:commons-lang3:${versions.commonslang}"
+
+    integrationTestImplementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+    integrationTestImplementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+
+    integrationTestImplementation "org.opensearch:opensearch:${opensearch_version}"
+}
+
+def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile
+es_tmp_dir.mkdirs()
+
+File repo = file("$buildDir/testclusters/repo")
+def _numNodes = findProperty('numNodes') as Integer ?: 1
+
+licenseHeaders.enabled = true
+validateNebulaPom.enabled = false
+testingConventions.enabled = false
+loggerUsageCheck.enabled = false
+
+javaRestTest.dependsOn(rootProject.assemble)
+javaRestTest {
+    systemProperty 'tests.security.manager', 'false'
+}
+testClusters.javaRestTest {
+    testDistribution = 'INTEG_TEST'
+}
+
+task integTest(type: RestIntegTestTask) {
+    description = "Run tests against a cluster"
+    testClassesDirs = sourceSets.test.output.classesDirs
+    classpath = sourceSets.test.runtimeClasspath
+}
+tasks.named("check").configure { dependsOn(integTest) }
+
+//create source set 'integrationTest'
+//add classes from the main source set to the compilation and runtime classpaths of the integrationTest
+sourceSets {
+    integrationTest {
+        java {
+            srcDir file ('src/integrationTest/java')
+            compileClasspath += sourceSets.main.output
+            runtimeClasspath += sourceSets.main.output
+        }
+        resources {
+            srcDir file('src/integrationTest/resources')
+        }
+        processIntegrationTestResources {
+            duplicatesStrategy(DuplicatesStrategy.INCLUDE)
+        }
+    }
+}
+
+task integrationTest(type: Test) {
+    testClassesDirs = sourceSets.integrationTest.output.classesDirs
+    classpath = sourceSets.integrationTest.runtimeClasspath
+}
+
+configurations {
+    integrationTestImplementation.extendsFrom(rootProject.configurations.implementation)
+    integrationTestImplementation.extendsFrom(rootProject.configurations.testImplementation)
+    integrationTestImplementation.extendsFrom(rootProject.configurations.integrationTestImplementation)
+}
+
+integTest {
+    if (project.hasProperty('excludeTests')) {
+        project.properties['excludeTests']?.replaceAll('\\s', '')?.split('[,;]')?.each {
+            exclude "${it}"
+        }
+    }
+    systemProperty 'tests.security.manager', 'false'
+    systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath
+
+    systemProperty "https", System.getProperty("https")
+    systemProperty "user", System.getProperty("user")
+    systemProperty "password", System.getProperty("password")
+    // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
+    // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
+    doFirst {
+        // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
+        // use longer timeouts for requests.
+        def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
+        systemProperty 'cluster.debug', isDebuggingCluster
+        // Set number of nodes system property to be used in tests
+        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
+        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
+        // not being written, the waitForAllConditions ensures it's written
+        getClusters().forEach { cluster ->
+            cluster.waitForAllConditions()
+        }
+    }
+
+    // The -Dcluster.debug option makes the cluster debuggable; this makes the tests debuggable
+    if (System.getProperty("test.debug") != null) {
+        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000'
+    }
+}
+project.getTasks().getByName('bundlePlugin').dependsOn(rootProject.tasks.getByName('build'))
+Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
+Zip rootBundle = (Zip) rootProject.getTasks().getByName("bundlePlugin");
+integTest.dependsOn(bundle)
+integTest.getClusters().forEach{c -> {
+    c.plugin(rootProject.getObjects().fileProperty().value(rootBundle.getArchiveFile()))
+    c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))
+}}
+
+testClusters.integTest {
+    testDistribution = 'INTEG_TEST'
+
+    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
+    if (_numNodes > 1) numberOfNodes = _numNodes
+    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
+    // i.e. we have to use a custom property to flag when we want to debug OpenSearch JVM
+    // since we also support multi node integration tests we increase debugPort per node
+    if (System.getProperty("cluster.debug") != null) {
+        def debugPort = 5005
+        nodes.forEach { node ->
+            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
+            debugPort += 1
+        }
+    }
+    setting 'path.repo', repo.absolutePath
+}
+
+afterEvaluate {
+    testClusters.integTest.nodes.each { node ->
+        def plugins = node.plugins
+        def firstPlugin = plugins.get(0)
+        if (firstPlugin.provider == project.bundlePlugin.archiveFile) {
+            plugins.remove(0)
+            plugins.add(firstPlugin)
+        }
+
+        node.extraConfigFile("kirk.pem", rootProject.file("bwc-test/src/test/resources/security/kirk.pem"))
+        node.extraConfigFile("kirk-key.pem", rootProject.file("bwc-test/src/test/resources/security/kirk-key.pem"))
+        node.extraConfigFile("esnode.pem", rootProject.file("bwc-test/src/test/resources/security/esnode.pem"))
+        node.extraConfigFile("esnode-key.pem", rootProject.file("bwc-test/src/test/resources/security/esnode-key.pem"))
+        node.extraConfigFile("root-ca.pem", rootProject.file("bwc-test/src/test/resources/security/root-ca.pem"))
+        node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
+        node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
+        node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
+        node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
+        node.setting("plugins.security.ssl.http.enabled", "true")
+        node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
+        node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
+        node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+        node.setting("plugins.security.allow_unsafe_democertificates", "true")
+        node.setting("plugins.security.allow_default_init_securityindex", "true")
+        node.setting("plugins.security.authcz.admin_dn", "\n - CN=kirk,OU=client,O=client,L=test,C=de")
+        node.setting("plugins.security.audit.type", "internal_opensearch")
+        node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
+        node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
+        node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
+    }
+}
+
+run {
+    doFirst {
+        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
+        // not being written, the waitForAllConditions ensures it's written
+        getClusters().forEach { cluster ->
+            cluster.waitForAllConditions()
+        }
+    }
+    useCluster testClusters.integTest
+}
+
+// As of ES 7.7 the sample-extension-plugin is being added to the list of plugins for the testCluster during build before
+// the security plugin is causing build failures.
+// The security zip is added explicitly above but the sample-extension-plugin is added implicitly at some time during evaluation.
+// Will need to do a deep dive to find out exactly what task adds the sample-extension-plugin and add security there but a temporary hack is to
+// reorder the plugins list after evaluation but prior to task execution when the plugins are installed.
+//afterEvaluate {
+//    testClusters.javaRestTest.nodes.each { node ->
+//        def nodePlugins = node.plugins
+//        def firstPlugin = nodePlugins.get(0)
+//        if (firstPlugin.provider == project.bundlePlugin.archiveFile) {
+//            nodePlugins.remove(0)
+//            nodePlugins.add(firstPlugin)
+//        }
+//    }
+//}