From 5cc2cfc259901fda60cf6d878a1169ce0240f8b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Feb 2025 11:00:26 -0800 Subject: [PATCH 1/3] dependabot: bump com.netflix.nebula.ospackage from 11.5.0 to 11.11.1 (#1422) Bumps com.netflix.nebula.ospackage from 11.5.0 to 11.11.1. --- updated-dependencies: - dependency-name: com.netflix.nebula.ospackage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f81bec659..00ba17793 100644 --- a/build.gradle +++ b/build.gradle @@ -65,7 +65,7 @@ buildscript { } plugins { - id 'com.netflix.nebula.ospackage' version "11.5.0" + id 'com.netflix.nebula.ospackage' version "11.11.1" id "com.diffplug.spotless" version "6.25.0" id 'java-library' id 'org.gradle.test-retry' version '1.6.0' From 5c1b0442e65c0f036fb1b82fcb21b798f060177f Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 18 Mar 2025 16:13:22 -0400 Subject: [PATCH 2/3] Use testclusters when testing with security (#1414) * Use testclusters when testing with security Signed-off-by: Craig Perkins * Add download plugin Signed-off-by: Craig Perkins * Get js and security plugin Signed-off-by: Craig Perkins * Add opensearchPlugin Signed-off-by: Craig Perkins * Remove duplicate Signed-off-by: Craig Perkins * Wait for yellow Signed-off-by: Craig Perkins * Fix tests Signed-off-by: Craig Perkins * Fix bwc test Signed-off-by: Craig Perkins * Add prepareBwcTests Signed-off-by: Craig Perkins * Add to developer guide Signed-off-by: Craig Perkins * Add to CHANGELOG Signed-off-by: Craig Perkins --------- Signed-off-by: Craig Perkins --- .github/workflows/test_security.yml | 100 ++---- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 1 + build.gradle | 335 +++++++++++++++--- .../timeseries/ODFERestTestCase.java | 14 +- 5 files changed, 320 insertions(+), 131 deletions(-) diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index f2275bf6b..75aaffa51 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -1,17 +1,20 @@ -name: Security test workflow for Anomaly Detection +name: Security test workflow +# This workflow is triggered on pull requests to main on: - push: - branches: - - "*" pull_request: branches: - - "*" - -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + - "**" + push: + branches: + - "**" jobs: + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main + with: + product: opensearch Build-ad: + needs: Get-CI-Image-Tag strategy: matrix: java: [11,17,21] @@ -19,67 +22,32 @@ jobs: name: Security test workflow for Anomaly Detection runs-on: ubuntu-latest + container: + # using the same image which is used by opensearch-build team to build the OpenSearch Distribution + # this image tag is subject to change as more dependencies and updates will arrive over time + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} steps: - - name: Setup Java ${{ matrix.java }} - uses: actions/setup-java@v3 + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + # This step uses the setup-java Github action: https://github.com/actions/setup-java + - name: Set Up JDK + uses: actions/setup-java@v4 with: - distribution: 'temurin' + distribution: temurin # Temurin is a distribution of adoptium java-version: ${{ matrix.java }} - - # time-series-analytics - - name: Checkout AD + # index-management + - name: Checkout Branch uses: actions/checkout@v4 - - - name: Build Anomaly Detection + - name: Run integration tests run: | - ./gradlew assemble - # example of variables: - # plugin = opensearch-time-series-analytics-2.10.0.0-SNAPSHOT.zip - # version = 2.10.0, plugin_version = 2.10.0.0, qualifier = SNAPSHOT - - name: Pull and Run Docker - run: | - plugin=`basename $(ls build/distributions/*.zip)` - version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3` - plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4` - qualifier=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1` - - if $qualifier!=SNAPSHOT - then - docker_version=$version-$qualifier - else - docker_version=$version - fi - echo plugin version plugin_version qualifier docker_version - echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)" - - cd .. - if docker pull opensearchstaging/opensearch:$docker_version - then - echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-anomaly-detection ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-anomaly-detection; fi" >> Dockerfile - echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-time-series-analytics ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-time-series-analytics; fi" >> Dockerfile - echo "ADD anomaly-detection/build/distributions/$plugin /tmp/" >> Dockerfile - echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile - docker build -t opensearch-ad:test . - echo "imagePresent=true" >> $GITHUB_ENV - else - echo "imagePresent=false" >> $GITHUB_ENV - fi - - name: Run Docker Image - if: env.imagePresent == 'true' - run: | - cd .. - docker run -p 9200:9200 -d -p 9600:9600 -e "OPENSEARCH_INITIAL_ADMIN_PASSWORD=myStrongPassword123!" -e "discovery.type=single-node" opensearch-ad:test - sleep 90 - - name: Run AD Test - if: env.imagePresent == 'true' - run: | - security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:myStrongPassword123! --insecure |grep opensearch-security|wc -l` - if [ $security -gt 0 ] - then - echo "Security plugin is available" - ./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=myStrongPassword123! - else - echo "Security plugin is NOT available, skipping integration tests" - fi + chown -R 1000:1000 `pwd` + su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'" + - name: Upload failed logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: logs + overwrite: 'true' + path: build/testclusters/integTest-*/logs/* diff --git a/CHANGELOG.md b/CHANGELOG.md index 02527e3f6..0200f3266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) ## [Unreleased 3.0](https://github.com/opensearch-project/anomaly-detection/compare/2.x...HEAD) ### Features ### Enhancements +- Use testclusters when testing with security ([#1414](https://github.com/opensearch-project/anomaly-detection/pull/1414)) ### Bug Fixes ### Infrastructure diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index a66823e27..70c405116 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -51,6 +51,7 @@ Currently we just put RCF jar in lib as dependency. Plan to publish to Maven and 10. `./gradlew bwcTestSuite -Dtests.security.manager=false` runs all the above bwc tests combined. 11. `./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true` launches HCAD model performance tests and logs the result in the standard output 12. `./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true` launches single stream AD model performance tests and logs the result in the standard output +13. `./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'` runs integration tests against a secure cluster When launching a cluster using one of the above commands logs are placed in `/build/cluster/run node0/opensearch-/logs`. Though the logs are teed to the console, in practices it's best to check the actual log file. diff --git a/build.gradle b/build.gradle index 00ba17793..751ba4a96 100644 --- a/build.gradle +++ b/build.gradle @@ -9,11 +9,28 @@ * GitHub history for details. */ + +import org.opensearch.gradle.testclusters.OpenSearchCluster + +import javax.net.ssl.HostnameVerifier +import javax.net.ssl.HttpsURLConnection +import javax.net.ssl.SSLContext +import javax.net.ssl.SSLSession +import javax.net.ssl.TrustManager +import javax.net.ssl.X509TrustManager +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.security.GeneralSecurityException +import java.security.cert.X509Certificate import java.util.concurrent.Callable import org.opensearch.gradle.test.RestIntegTestTask import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask import org.opensearch.gradle.info.BuildParams +import java.util.concurrent.TimeUnit +import java.util.function.Predicate +import java.util.stream.Collectors + buildscript { ext { opensearch_group = "org.opensearch" @@ -69,6 +86,7 @@ plugins { id "com.diffplug.spotless" version "6.25.0" id 'java-library' id 'org.gradle.test-retry' version '1.6.0' + id "de.undercouch.download" version "5.6.0" } tasks.withType(JavaCompile) { @@ -108,10 +126,10 @@ configurations { testImplementation { exclude group: 'org.hamcrest', module: 'hamcrest-core' } + opensearchPlugin } dependencies { - zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" implementation "org.opensearch:opensearch:${opensearch_version}" compileOnly "org.opensearch.plugin:opensearch-scripting-painless-spi:${opensearch_version}" compileOnly "org.opensearch:opensearch-job-scheduler-spi:${job_scheduler_version}" @@ -164,6 +182,9 @@ dependencies { testImplementation "org.opensearch:opensearch-core:${opensearch_version}" testRuntimeOnly("org.junit.platform:junit-platform-launcher:1.11.2") testCompileOnly 'junit:junit:4.13.2' + + opensearchPlugin "org.opensearch.plugin:opensearch-job-scheduler:${opensearch_build}@zip" + opensearchPlugin "org.opensearch.plugin:opensearch-security:${opensearch_build}@zip" } apply plugin: 'java' @@ -199,6 +220,18 @@ ext { projectSubstitutions = [:] licenseFile = rootProject.file('LICENSE.txt') noticeFile = rootProject.file('NOTICE.txt') + + ['esnode.pem', 'esnode-key.pem', 'kirk.pem', 'kirk-key.pem', 'root-ca.pem', 'sample.pem', 'test-kirk.jks'].forEach { file -> + File local = getLayout().getBuildDirectory().file(file).get().getAsFile() + download.run { + src "https://raw.githubusercontent.com/opensearch-project/security/refs/heads/main/bwc-test/src/test/resources/security/" + file + dest local + overwrite false + } + processResources { + from(local) + } + } } opensearchplugin { @@ -397,6 +430,29 @@ integTest { } } +ext.resolvePluginFile = { pluginId -> + return new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.opensearchPlugin.resolvedConfiguration.resolvedArtifacts + .find { ResolvedArtifact f -> + f.name.startsWith(pluginId) + } + .file + } + } + } + } +} +def jobSchedulerFile = resolvePluginFile("opensearch-job-scheduler") +def securityPluginFile = resolvePluginFile("opensearch-security") + +// === Setup security test === +// This flag indicates the existence of security plugin +def securityEnabled = System.getProperty("security", "false") == "true" || System.getProperty("https", "false") == "true" testClusters.integTest { testDistribution = "ARCHIVE" // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 @@ -412,17 +468,10 @@ testClusters.integTest { } } plugin(project.tasks.bundlePlugin.archiveFile) - plugin(provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.getSingleFile() - } - } - } - })) + plugin(provider(jobSchedulerFile)) + if (securityEnabled) { + plugin(provider(securityPluginFile)) + } // As of ES 7.7.0 the opendistro-anomaly-detection plugin is being added to the list of plugins for the testCluster during build before // the opensearch-job-scheduler plugin, which is causing build failures. From the stack trace, this looks like a bug. @@ -446,6 +495,188 @@ testClusters.integTest { def firstPlugin = plugins.get(0) plugins.remove(0) plugins.add(firstPlugin) + + if (securityEnabled) { + node.extraConfigFile("kirk.pem", file("build/resources/main/kirk.pem")) + node.extraConfigFile("kirk-key.pem", file("build/resources/main/kirk-key.pem")) + node.extraConfigFile("esnode.pem", file("build/resources/main/esnode.pem")) + node.extraConfigFile("esnode-key.pem", file("build/resources/main/esnode-key.pem")) + node.extraConfigFile("root-ca.pem", file("build/resources/main/root-ca.pem")) + node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem") + node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem") + node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") + node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false") + node.setting("plugins.security.ssl.http.enabled", "true") + node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem") + node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem") + node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") + node.setting("plugins.security.allow_unsafe_democertificates", "true") + node.setting("plugins.security.allow_default_init_securityindex", "true") + node.setting("plugins.security.authcz.admin_dn", "\n - CN=kirk,OU=client,O=client,L=test,C=de") + node.setting("plugins.security.audit.type", "internal_opensearch") + node.setting("plugins.security.enable_snapshot_restore_privilege", "true") + node.setting("plugins.security.check_snapshot_restore_write_privileges", "true") + node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]") + node.setting("plugins.security.system_indices.enabled", "true") + // node.setting("plugins.security.system_indices.indices", "[\".opendistro-ism-config\"]") + } + } +} + +// Re-write WaitForHttpResource with updated code to support security plugin use case +class WaitForClusterYellow { + + private URL url + private String username + private String password + Set validResponseCodes = Collections.singleton(200) + + WaitForClusterYellow(String protocol, String host, int numberOfNodes) throws MalformedURLException { + this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow")) + } + + WaitForClusterYellow(URL url) { + this.url = url + } + + boolean wait(int durationInMs) throws GeneralSecurityException, InterruptedException, IOException { + final long waitUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationInMs) + final long sleep = 100 + + IOException failure = null + while (true) { + try { + checkResource() + return true + } catch (IOException e) { + failure = e + } + if (System.nanoTime() < waitUntil) { + Thread.sleep(sleep) + } else { + throw failure + } + } + } + + void setUsername(String username) { + this.username = username + } + + void setPassword(String password) { + this.password = password + } + + void checkResource() throws IOException { + final HttpURLConnection connection = buildConnection() + connection.connect() + final Integer response = connection.getResponseCode() + if (validResponseCodes.contains(response)) { + return + } else { + throw new IOException(response + " " + connection.getResponseMessage()) + } + } + + HttpURLConnection buildConnection() throws IOException { + final HttpURLConnection connection = (HttpURLConnection) this.@url.openConnection() + + if (connection instanceof HttpsURLConnection) { + TrustManager[] trustAllCerts = [new X509TrustManager() { + X509Certificate[] getAcceptedIssuers() { + return null + } + + void checkClientTrusted(X509Certificate[] certs, String authType) { + } + + void checkServerTrusted(X509Certificate[] certs, String authType) { + } + } + ] as TrustManager[] + SSLContext sc = SSLContext.getInstance("SSL") + sc.init(null, trustAllCerts, new java.security.SecureRandom()) + connection.setSSLSocketFactory(sc.getSocketFactory()) + // Create all-trusting host name verifier + HostnameVerifier allHostsValid = new HostnameVerifier() { + boolean verify(String hostname, SSLSession session) { + return true + } + } + // Install the all-trusting host verifier + connection.setHostnameVerifier(allHostsValid) + } + + configureBasicAuth(connection) + connection.setRequestMethod("GET") + return connection + } + + void configureBasicAuth(HttpURLConnection connection) { + if (username != null) { + if (password == null) { + throw new IllegalStateException("Basic Auth user [" + username + "] has been set, but no password has been configured") + } + connection.setRequestProperty( + "Authorization", + "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8)) + ) + } + } + +} + +def waitForClusterSetup(OpenSearchCluster cluster, Boolean securityEnabled) { + cluster.@waitConditions.clear() + String unicastUris = cluster.nodes.stream().flatMap { node -> + node.getAllTransportPortURI().stream() + }.collect(Collectors.joining("\n")) + cluster.nodes.forEach { node -> + try { + Files.write(node.getConfigDir().resolve("unicast_hosts.txt"), unicastUris.getBytes(StandardCharsets.UTF_8)) + } catch (IOException e) { + throw new java.io.UncheckedIOException("Failed to write configuation files for " + this, e) + } + } + + Predicate pred = { + String protocol = securityEnabled ? "https" : "http" + String host = System.getProperty("tests.cluster", cluster.getFirstNode().getHttpSocketURI()) + WaitForClusterYellow wait = new WaitForClusterYellow(protocol, host, cluster.nodes.size()) + wait.setUsername(System.getProperty("user", "admin")) + wait.setPassword(System.getProperty("password", "admin")) + return wait.wait(180000) + } + + cluster.@waitConditions.put("cluster health yellow", pred) + cluster.waitForAllConditions() +} + +integTest { + systemProperty 'tests.security.manager', 'false' + systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath + systemProperty 'buildDir', buildDir.path + systemProperty "https", System.getProperty("https") + systemProperty "security", System.getProperty("security") + systemProperty "user", System.getProperty("user", "admin") + systemProperty "password", "admin" + // defaulting to admin since security plugin's demo config tool is not used + // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for + // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time. + doFirst { + systemProperty 'cluster.debug', getDebug() + // Set number of nodes system property to be used in tests + systemProperty 'cluster.number_of_nodes', "${_numNodes}" + // There seems to be an issue when running multi node run or integ tasks with unicast_hosts + // not being written, the waitForAllConditions ensures it's written + getClusters().forEach { cluster -> + waitForClusterSetup(cluster, securityEnabled) + } + } + + // The -Dcluster.debug option makes the cluster debuggable, this makes the tests debuggable + if (System.getProperty("test.debug") != null) { + jvmArgs '-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=8000' } } @@ -475,30 +706,36 @@ task integTestRemote(type: RestIntegTestTask) { } } +configurations { + bwcZip +} +dependencies { + bwcZip "org.opensearch.plugin:opensearch-job-scheduler:${bwcVersion}-SNAPSHOT@zip" +} +ext.resolvebwcZipFile = { pluginId -> + return new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return configurations.bwcZip.resolvedConfiguration.resolvedArtifacts + .find { ResolvedArtifact f -> + f.name.startsWith(pluginId) + } + .file + } + } + } + } +} 2.times {i -> testClusters { "${baseName}$i" { testDistribution = "ARCHIVE" versions = [bwcVersionShort, opensearch_version] numberOfNodes = 3 - plugin(provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - if (new File("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion").exists()) { - project.delete(files("$project.rootDir/$bwcFilePath/job-scheduler/$bwcVersion")) - } - project.mkdir bwcJobSchedulerPath + bwcVersion - ant.get(src: bwcOpenSearchJSDownload, - dest: bwcJobSchedulerPath + bwcVersion, - httpusecaches: false) - return fileTree(bwcJobSchedulerPath + bwcVersion).getSingleFile() - } - } - } - })) + plugin(provider(resolvebwcZipFile("opensearch-job-scheduler"))) plugin(provider(new Callable(){ @Override RegularFile call() throws Exception { @@ -523,34 +760,24 @@ task integTestRemote(type: RestIntegTestTask) { } } -List> plugins = [ - provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return configurations.zipArchive.asFileTree.getSingleFile() - } - } - } - }), - provider(new Callable(){ - @Override - RegularFile call() throws Exception { - return new RegularFile() { - @Override - File getAsFile() { - return fileTree(bwcFilePath + "anomaly-detection/" + project.version).getSingleFile() - } - } - } - }) - ] +List> plugins = [] + +// Ensure the artifact for the current project version is available to be used for the bwc tests + +task prepareBwcTests { + dependsOn bundlePlugin + doLast { + plugins = [ + provider(jobSchedulerFile), + project.getObjects().fileProperty().value(project.tasks.bundlePlugin.archiveFile) + ] + } +} // Creates 2 test clusters with 3 nodes of the old version. 2.times {i -> task "${baseName}#oldVersionClusterTask$i"(type: RestIntegTestTask) { + dependsOn 'prepareBwcTests' useCluster testClusters."${baseName}$i" filter { includeTestsMatching "org.opensearch.ad.bwc.*IT" diff --git a/src/test/java/org/opensearch/timeseries/ODFERestTestCase.java b/src/test/java/org/opensearch/timeseries/ODFERestTestCase.java index 6d0f6c290..0a4a3ed01 100644 --- a/src/test/java/org/opensearch/timeseries/ODFERestTestCase.java +++ b/src/test/java/org/opensearch/timeseries/ODFERestTestCase.java @@ -83,15 +83,7 @@ public abstract class ODFERestTestCase extends OpenSearchRestTestCase { private static final Logger LOG = (Logger) LogManager.getLogger(ODFERestTestCase.class); protected boolean isHttps() { - boolean isHttps = Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); - if (isHttps) { - // currently only external cluster is supported for security enabled testing - if (!Optional.ofNullable(System.getProperty("tests.rest.cluster")).isPresent()) { - throw new RuntimeException("cluster url should be provided for security enabled testing"); - } - } - - return isHttps; + return Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false); } @Override @@ -138,12 +130,12 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE if (Objects.nonNull(keystore)) { URI uri = null; try { - uri = this.getClass().getClassLoader().getResource("security/sample.pem").toURI(); + uri = this.getClass().getClassLoader().getResource("sample.pem").toURI(); } catch (URISyntaxException e) { throw new RuntimeException(e); } Path configPath = PathUtils.get(uri).getParent().toAbsolutePath(); - return new SecureRestClientBuilder(settings, configPath).build(); + return new SecureRestClientBuilder(settings, configPath, hosts).build(); } else { configureHttpsClient(builder, settings); builder.setStrictDeprecationMode(strictDeprecationMode); From 5718cde0dfb1bda0b9033fc856d48a55e749646d Mon Sep 17 00:00:00 2001 From: Amit Galitzky Date: Wed, 19 Mar 2025 10:30:35 -0700 Subject: [PATCH 3/3] adding ability to run AD with 2 local clusters (#1441) Signed-off-by: Amit Galitzky --- .github/workflows/changelog_verifier.yml | 1 + .github/workflows/labeler.yml | 3 - .github/workflows/test_security.yml | 2 +- CHANGELOG.md | 4 +- DEVELOPER_GUIDE.md | 24 +-- build.gradle | 193 +++++++++++++++-------- 6 files changed, 148 insertions(+), 79 deletions(-) diff --git a/.github/workflows/changelog_verifier.yml b/.github/workflows/changelog_verifier.yml index 387176f63..af774a29f 100644 --- a/.github/workflows/changelog_verifier.yml +++ b/.github/workflows/changelog_verifier.yml @@ -4,6 +4,7 @@ on: branches-ignore: - 'whitesource-remediate/**' - 'backport/**' + - 'dependabot/**' pull_request: types: [opened, edited, review_requested, synchronize, reopened, ready_for_review, labeled, unlabeled] diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index b7bf3fd8b..910df1d3b 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -6,9 +6,6 @@ on: types: - opened -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - jobs: label: runs-on: ubuntu-latest diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml index 75aaffa51..50a8b71b0 100644 --- a/.github/workflows/test_security.yml +++ b/.github/workflows/test_security.yml @@ -50,4 +50,4 @@ jobs: with: name: logs overwrite: 'true' - path: build/testclusters/integTest-*/logs/* + path: build/testclusters/integTest-*/logs/* \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0200f3266..0ad299f95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) - Use testclusters when testing with security ([#1414](https://github.com/opensearch-project/anomaly-detection/pull/1414)) ### Bug Fixes ### Infrastructure - +- Adding dual cluster arg to gradle run ([#1441](https://github.com/opensearch-project/anomaly-detection/pull/1441)) ### Documentation ### Maintenance @@ -21,9 +21,11 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) ### Enhancements - Github workflow for changelog verification ([#1413](https://github.com/opensearch-project/anomaly-detection/pull/1413)) +- Use testclusters when testing with security ([#1414](https://github.com/opensearch-project/anomaly-detection/pull/1414)) ### Bug Fixes ### Infrastructure +- Adding dual cluster arg to gradle run ([#1441](https://github.com/opensearch-project/anomaly-detection/pull/1441)) ### Documentation ### Maintenance ### Refactoring \ No newline at end of file diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 70c405116..090ab4bba 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -41,17 +41,19 @@ Currently we just put RCF jar in lib as dependency. Plan to publish to Maven and 1. `./gradlew build` builds and tests 2. `./gradlew :run` launches a single node cluster with anomaly-detection (and job-scheduler) plugin installed -3. `./gradlew :integTest` launches a single node cluster with anomaly-detection (and job-scheduler) plugin installed and runs all integration tests except security -4. ` ./gradlew :integTest --tests="**.test execute foo"` runs a single integration test class or method -5. `./gradlew integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=` launches integration tests against a local cluster and run tests with security -6. `./gradlew spotlessApply` formats code. And/or import formatting rules in `.eclipseformat.xml` with IDE. -7. `./gradlew adBwcCluster#mixedClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by upgrading one of the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler creating a mixed cluster. -8. `./gradlew adBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing rolling upgrade of each node with the current version of OpenSearch with anomaly-detection and job-scheduler. -9. `./gradlew adBwcCluster#fullRestartClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing a full restart on the cluster upgrading all the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler. -10. `./gradlew bwcTestSuite -Dtests.security.manager=false` runs all the above bwc tests combined. -11. `./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true` launches HCAD model performance tests and logs the result in the standard output -12. `./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true` launches single stream AD model performance tests and logs the result in the standard output -13. `./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'` runs integration tests against a secure cluster +3. `./gradlew :run -PdualCluster=true` launches 2 single node clusters with anomaly-detection (and job-scheduler) plugin installed, one cluster is on localhost:9200 and the other on localhost:9200 +4. `./gradlew :integTest` launches a single node cluster with anomaly-detection (and job-scheduler) plugin installed and runs all integration tests except security +5. ` ./gradlew :integTest --tests="**.test execute foo"` runs a single integration test class or method +6. `./gradlew integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=` launches integration tests against a local cluster and run tests with security +7. `./gradlew spotlessApply` formats code. And/or import formatting rules in `.eclipseformat.xml` with IDE. +8. `./gradlew adBwcCluster#mixedClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by upgrading one of the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler creating a mixed cluster. +9. `./gradlew adBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing rolling upgrade of each node with the current version of OpenSearch with anomaly-detection and job-scheduler. +10. `./gradlew adBwcCluster#fullRestartClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with anomaly-detection and job-scheduler and tests backwards compatibility by performing a full restart on the cluster upgrading all the nodes with the current version of OpenSearch with anomaly-detection and job-scheduler. +11. `./gradlew bwcTestSuite -Dtests.security.manager=false` runs all the above bwc tests combined. +12. `./gradlew ':test' --tests "org.opensearch.ad.ml.HCADModelPerfTests" -Dtests.seed=2AEBDBBAE75AC5E0 -Dtests.security.manager=false -Dtests.locale=es-CU -Dtests.timezone=Chile/EasterIsland -Dtest.logs=true -Dmodel-benchmark=true` launches HCAD model performance tests and logs the result in the standard output +13. `./gradlew integTest --tests "org.opensearch.ad.e2e.SingleStreamModelPerfIT" -Dtests.seed=60CDDB34427ACD0C -Dtests.security.manager=false -Dtests.locale=kab-DZ -Dtests.timezone=Asia/Hebron -Dtest.logs=true -Dmodel-benchmark=true` launches single stream AD model performance tests and logs the result in the standard output +14. `./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'` runs integration tests against a secure cluster + When launching a cluster using one of the above commands logs are placed in `/build/cluster/run node0/opensearch-/logs`. Though the logs are teed to the console, in practices it's best to check the actual log file. diff --git a/build.gradle b/build.gradle index 751ba4a96..c7736f4a2 100644 --- a/build.gradle +++ b/build.gradle @@ -26,6 +26,8 @@ import java.util.concurrent.Callable import org.opensearch.gradle.test.RestIntegTestTask import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask import org.opensearch.gradle.info.BuildParams +import java.util.stream.Collectors + import java.util.concurrent.TimeUnit import java.util.function.Predicate @@ -65,7 +67,7 @@ buildscript { jacksonVersion = "2.18.2" // gradle build won't print logs during test by default unless there is a failure. // It is useful to record intermediately information like prediction precision and recall. - // This option turn on log printing during tests. + // This option turn on log printing during tests. printLogs = "true" == System.getProperty("test.logs", "false") } @@ -260,7 +262,7 @@ configurations.all { force "com.google.guava:guava:32.1.3-jre" // CVE for 31.1 force("com.fasterxml.jackson.core:jackson-core:${jacksonVersion}") - force "org.eclipse.platform:org.eclipse.core.runtime:3.29.0" // CVE for < 3.29.0 + force "org.eclipse.platform:org.eclipse.core.runtime:3.29.0" // CVE for < 3.29.0 } } @@ -287,7 +289,7 @@ publishing { } } } - + repositories { maven { name = "Snapshots" @@ -453,10 +455,35 @@ def securityPluginFile = resolvePluginFile("opensearch-security") // === Setup security test === // This flag indicates the existence of security plugin def securityEnabled = System.getProperty("security", "false") == "true" || System.getProperty("https", "false") == "true" + testClusters.integTest { testDistribution = "ARCHIVE" // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes + configureClusterPlugins(delegate, jobSchedulerFile, securityPluginFile, securityEnabled) +} + +testClusters { + leaderCluster { + testDistribution = "ARCHIVE" + setting 'path.repo', file("${buildDir}/leaderCluster/repo").absolutePath + setting 'discovery.type', 'single-node' + // Add custom attribute to identify this cluster + setting 'node.attr.cluster_role', 'leader' + configureClusterPlugins(delegate, jobSchedulerFile, securityPluginFile, securityEnabled) + } + + followCluster { + testDistribution = "ARCHIVE" + setting 'path.repo', file("${buildDir}/followCluster/repo").absolutePath + setting 'discovery.type', 'single-node' + // Add custom attribute to identify this cluster + setting 'node.attr.cluster_role', 'follower' + configureClusterPlugins(delegate, jobSchedulerFile, securityPluginFile, securityEnabled) + } +} + +def configureClusterPlugins(cluster, jobSchedulerFile, securityPluginFile, securityEnabled) { // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM // since we also support multi node integration tests we increase debugPort per node @@ -467,58 +494,60 @@ testClusters.integTest { debugPort += 1 } } - plugin(project.tasks.bundlePlugin.archiveFile) - plugin(provider(jobSchedulerFile)) - if (securityEnabled) { - plugin(provider(securityPluginFile)) - } - - // As of ES 7.7.0 the opendistro-anomaly-detection plugin is being added to the list of plugins for the testCluster during build before - // the opensearch-job-scheduler plugin, which is causing build failures. From the stack trace, this looks like a bug. - // - // Exception in thread "main" java.lang.IllegalArgumentException: Missing plugin [opensearch-job-scheduler], dependency of [opendistro-anomaly-detection] - // at org.opensearch.plugins.PluginsService.addSortedBundle(PluginsService.java:452) - // - // One explanation is that ES build script sort plugins according to the natural ordering of their names. - // opendistro-anomaly-detection comes before opensearch-job-scheduler. - // - // The following is a comparison of different plugin installation order: - // Before 7.7: - // ./bin/elasticsearch-plugin install --batch file:opendistro-anomaly-detection.zip file:opensearch-job-scheduler.zip - // - // After 7.7: - // ./bin/elasticsearch-plugin install --batch file:opensearch-job-scheduler.zip file:opendistro-anomaly-detection.zip - // - // A temporary hack is to reorder the plugins list after evaluation but prior to task execution when the plugins are installed. - nodes.each { node -> - def plugins = node.plugins - def firstPlugin = plugins.get(0) - plugins.remove(0) - plugins.add(firstPlugin) - + cluster.with { + plugin(project.tasks.bundlePlugin.archiveFile) + plugin(provider(jobSchedulerFile)) if (securityEnabled) { - node.extraConfigFile("kirk.pem", file("build/resources/main/kirk.pem")) - node.extraConfigFile("kirk-key.pem", file("build/resources/main/kirk-key.pem")) - node.extraConfigFile("esnode.pem", file("build/resources/main/esnode.pem")) - node.extraConfigFile("esnode-key.pem", file("build/resources/main/esnode-key.pem")) - node.extraConfigFile("root-ca.pem", file("build/resources/main/root-ca.pem")) - node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem") - node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem") - node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") - node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false") - node.setting("plugins.security.ssl.http.enabled", "true") - node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem") - node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem") - node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") - node.setting("plugins.security.allow_unsafe_democertificates", "true") - node.setting("plugins.security.allow_default_init_securityindex", "true") - node.setting("plugins.security.authcz.admin_dn", "\n - CN=kirk,OU=client,O=client,L=test,C=de") - node.setting("plugins.security.audit.type", "internal_opensearch") - node.setting("plugins.security.enable_snapshot_restore_privilege", "true") - node.setting("plugins.security.check_snapshot_restore_write_privileges", "true") - node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]") - node.setting("plugins.security.system_indices.enabled", "true") - // node.setting("plugins.security.system_indices.indices", "[\".opendistro-ism-config\"]") + plugin(provider(securityPluginFile)) + } + + // As of ES 7.7.0 the opendistro-anomaly-detection plugin is being added to the list of plugins for the testCluster during build before + // the opensearch-job-scheduler plugin, which is causing build failures. From the stack trace, this looks like a bug. + // + // Exception in thread "main" java.lang.IllegalArgumentException: Missing plugin [opensearch-job-scheduler], dependency of [opendistro-anomaly-detection] + // at org.opensearch.plugins.PluginsService.addSortedBundle(PluginsService.java:452) + // + // One explanation is that ES build script sort plugins according to the natural ordering of their names. + // opendistro-anomaly-detection comes before opensearch-job-scheduler. + // + // The following is a comparison of different plugin installation order: + // Before 7.7: + // ./bin/elasticsearch-plugin install --batch file:opendistro-anomaly-detection.zip file:opensearch-job-scheduler.zip + // + // After 7.7: + // ./bin/elasticsearch-plugin install --batch file:opensearch-job-scheduler.zip file:opendistro-anomaly-detection.zip + // + // A temporary hack is to reorder the plugins list after evaluation but prior to task execution when the plugins are installed. + nodes.each { node -> + def plugins = node.plugins + def firstPlugin = plugins.get(0) + plugins.remove(0) + plugins.add(firstPlugin) + + if (securityEnabled) { + node.extraConfigFile("kirk.pem", file("build/resources/main/kirk.pem")) + node.extraConfigFile("kirk-key.pem", file("build/resources/main/kirk-key.pem")) + node.extraConfigFile("esnode.pem", file("build/resources/main/esnode.pem")) + node.extraConfigFile("esnode-key.pem", file("build/resources/main/esnode-key.pem")) + node.extraConfigFile("root-ca.pem", file("build/resources/main/root-ca.pem")) + node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem") + node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem") + node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem") + node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false") + node.setting("plugins.security.ssl.http.enabled", "true") + node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem") + node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem") + node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem") + node.setting("plugins.security.allow_unsafe_democertificates", "true") + node.setting("plugins.security.allow_default_init_securityindex", "true") + node.setting("plugins.security.authcz.admin_dn", "\n - CN=kirk,OU=client,O=client,L=test,C=de") + node.setting("plugins.security.audit.type", "internal_opensearch") + node.setting("plugins.security.enable_snapshot_restore_privilege", "true") + node.setting("plugins.security.check_snapshot_restore_write_privileges", "true") + node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]") + node.setting("plugins.security.system_indices.enabled", "true") + // node.setting("plugins.security.system_indices.indices", "[\".opendistro-ism-config\"]") + } } } } @@ -729,7 +758,7 @@ ext.resolvebwcZipFile = { pluginId -> } } } -2.times {i -> +2.times {i -> testClusters { "${baseName}$i" { testDistribution = "ARCHIVE" @@ -774,8 +803,8 @@ task prepareBwcTests { } } -// Creates 2 test clusters with 3 nodes of the old version. -2.times {i -> +// Creates 2 test clusters with 3 nodes of the old version. +2.times {i -> task "${baseName}#oldVersionClusterTask$i"(type: RestIntegTestTask) { dependsOn 'prepareBwcTests' useCluster testClusters."${baseName}$i" @@ -787,10 +816,10 @@ task prepareBwcTests { systemProperty 'tests.plugin_bwc_version', bwcVersion nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}") - } + } } -// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version +// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. // This is also used as a one third upgraded cluster for a rolling upgrade. task "${baseName}#mixedClusterTask"(type: RestIntegTestTask) { @@ -848,7 +877,7 @@ task "${baseName}#rollingUpgradeClusterTask"(type: RestIntegTestTask) { nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") } -// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version +// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version // at the same time resulting in a fully upgraded cluster. task "${baseName}#fullRestartClusterTask"(type: RestIntegTestTask) { dependsOn "${baseName}#oldVersionClusterTask1" @@ -876,16 +905,54 @@ task bwcTestSuite(type: RestIntegTestTask) { run { doFirst { - // There seems to be an issue when running multi node run or integ tasks with unicast_hosts - // not being written, the waitForAllConditions ensures it's written getClusters().forEach { cluster -> cluster.waitForAllConditions() } + + if (project.hasProperty('dualCluster')) { + def clusterPorts = [:] // Map to store leader/follow cluster info + + getClusters().forEach { cluster -> + def clusterName = cluster.getName() + def allTransportSocketURI = cluster.nodes + .collectMany { node -> node.getAllTransportPortURI() } + .join(",") + + def allHttpSocketURI = cluster.nodes + .collectMany { node -> node.getAllHttpSocketURI() } + .join(",") + + clusterPorts[clusterName] = [ + httpHosts: allHttpSocketURI, + transportHosts: allTransportSocketURI + ] + + println "${clusterName.capitalize()} cluster running at HTTP: ${allHttpSocketURI}, Transport: ${allTransportSocketURI}" + } + + // Pass cluster info to tests + if (clusterPorts.containsKey('leader') && clusterPorts.containsKey('follow')) { + systemProperty "tests.leader.cluster.name", "leader" + systemProperty "tests.follow.cluster.name", "follower" + + systemProperty "tests.cluster.leader.http_hosts", clusterPorts['leader'].httpHosts + systemProperty "tests.cluster.follow.http_hosts", clusterPorts['follow'].httpHosts + + systemProperty "tests.cluster.leader.transport_hosts", clusterPorts['leader'].transportHosts + systemProperty "tests.cluster.follow.transport_hosts", clusterPorts['follow'].transportHosts + } + } } - useCluster testClusters.integTest + if (project.hasProperty('dualCluster')) { + useCluster testClusters.leaderCluster + useCluster testClusters.followCluster + } else { + useCluster testClusters.integTest + } } + evaluationDependsOnChildren() task release(type: Copy, group: 'build') {