build.gradle

/*
 * Copyright OpenSearch Contributors
 * SPDX-License-Identifier: Apache-2.0
 */

import java.util.concurrent.Callable
import org.opensearch.gradle.test.RestIntegTestTask
import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask

plugins {
    id 'java'
    id 'com.netflix.nebula.ospackage'
    id "io.freefair.lombok"
    id 'jacoco'
    id 'java-library'
    id 'com.diffplug.spotless' version '6.25.0'
}

ext {
    opensearchVersion = "${opensearch_version}"
    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
}

apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.testclusters'
apply plugin: 'opensearch.pluginzip'
apply plugin: 'opensearch.yaml-rest-test'
ext {
    projectSubstitutions = [:]
    licenseFile = rootProject.file('LICENSE.txt')
    noticeFile = rootProject.file('NOTICE')
}

java {
    targetCompatibility = JavaVersion.VERSION_21
    sourceCompatibility = JavaVersion.VERSION_21
}

lombok {
    version = "1.18.30"
}

opensearchplugin {
    name 'opensearch-ml'
    description 'machine learning plugin for opensearch'
    classname 'org.opensearch.ml.plugin.MachineLearningPlugin'
    extendedPlugins = ['opensearch-job-scheduler']
}

configurations {
    zipArchive
}

dependencies {
    implementation project(path: ":${rootProject.name}-spi", configuration: 'shadow')
    implementation project(path: ":${rootProject.name}-common", configuration: 'shadow')
    implementation project(':opensearch-ml-algorithms')
    implementation project(':opensearch-ml-search-processors')
    implementation project(':opensearch-ml-memory')
    compileOnly "com.google.guava:guava:32.1.3-jre"

    implementation group: 'software.amazon.awssdk', name: 'aws-core', version: "2.30.18"
    implementation group: 'software.amazon.awssdk', name: 's3', version: "2.30.18"
    implementation group: 'software.amazon.awssdk', name: 'regions', version: "2.30.18"

    implementation group: 'software.amazon.awssdk', name: 'aws-xml-protocol', version: "2.30.18"

    implementation group: 'software.amazon.awssdk', name: 'aws-query-protocol', version: "2.30.18"

    implementation group: 'software.amazon.awssdk', name: 'protocol-core', version: "2.30.18"

    zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}"
    compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"
    implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}"
    implementation "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
    // Multi-tenant SDK Client
    implementation "org.opensearch:opensearch-remote-metadata-sdk:${opensearch_build}"

    implementation "org.opensearch:common-utils:${common_utils_version}"
    implementation("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
    implementation("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
    implementation group: 'com.networknt' , name: 'json-schema-validator', version: '1.4.0'
    implementation (group: 'com.google.guava', name: 'guava', version: '32.1.3-jre') {
	exclude group: 'com.google.errorprone', module: 'error_prone_annotations'
    }
    implementation group: 'com.google.code.gson', name: 'gson', version: '2.11.0'
    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.10'
    implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
    implementation group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
    implementation "org.apache.logging.log4j:log4j-slf4j-impl:2.19.0"
    testImplementation group: 'commons-io', name: 'commons-io', version: '2.15.1'
    implementation group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
    implementation ('com.jayway.jsonpath:json-path:2.9.0') {
        exclude group: 'net.minidev', module: 'json-smart'
    }
    implementation('net.minidev:json-smart:2.5.2')
}

publishing {
    publications {
        pluginZip(MavenPublication) { publication ->
            pom {
                name = opensearchplugin.name
                description = opensearchplugin.description
                groupId = "org.opensearch.plugin"
                licenses {
                    license {
                        name = "The Apache License, Version 2.0"
                        url = "http://www.apache.org/licenses/LICENSE-2.0.txt"
                    }
                }
                developers {
                    developer {
                        name = "OpenSearch"
                        url = "https://github.com/opensearch-project/ml-commons"
                    }
                }
            }
        }
    }

    repositories {
        maven {
            mavenLocal()
            mavenCentral()
            maven { url "https://ci.opensearch.org/ci/dbc/snapshots/lucene/" }
            name = "Snapshots"
            url = "https://aws.oss.sonatype.org/content/repositories/snapshots"
            credentials {
                username "$System.env.SONATYPE_USERNAME"
                password "$System.env.SONATYPE_PASSWORD"
            }
        }
    }
}

compileJava {
    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
}

compileTestJava {
    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
}

//TODO: check which one should be enabled
licenseHeaders.enabled = true
testingConventions.enabled = false
forbiddenApis.ignoreFailures = false
dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
forbiddenApisTest.ignoreFailures = true
forbiddenApisMain.ignoreFailures = true
validateNebulaPom.enabled = false
loggerUsageCheck.enabled = false

def _numNodes = findProperty('numNodes') as Integer ?: 1


test {
    include '**/*Tests.class'
    systemProperty 'tests.security.manager', 'false'
}

def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()


task integTest(type: RestIntegTestTask) {
    description = "Run tests against a cluster"
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
}
tasks.named("check").configure { dependsOn(integTest) }

integTest {
    dependsOn "bundlePlugin"
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    // Only rest case can run with remote cluster
    if (System.getProperty("tests.rest.cluster") != null) {
        filter {
            includeTestsMatching "org.opensearch.ml.rest.*IT"
            // mock LLM run in localhost, it will not reachable for docker or remote cluster
            excludeTestsMatching "org.opensearch.ml.tools.VisualizationsToolIT"
        }
    }

    if (System.getProperty("https") == null || System.getProperty("https") == "false") {
        filter {
            excludeTestsMatching "org.opensearch.ml.rest.SecureMLRestIT"
            excludeTestsMatching "org.opensearch.ml.rest.MLModelGroupRestIT"
        }
    }

    // BWC test can only run within the BWC gradle task bwcsuite or its dependent tasks.
    if (System.getProperty('tests.rest.bwcsuite') == null) {
        filter {
            excludeTestsMatching "org.opensearch.ml.bwc.*IT"
        }
    }

    // The 'doFirst' delays till execution time.
    doFirst {
        // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
        // use longer timeouts for requests.
        def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
        systemProperty 'cluster.debug', isDebuggingCluster
        // Set number of nodes system property to be used in tests
        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable
    if (System.getProperty("test.debug") != null) {
        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
    }

    // Set this to true this if you want to see the logs in the terminal test output.
    // note: if left false the log output will still show in your IDE
    testLogging.showStandardStreams = true
}

testClusters.integTest {
    testDistribution = "ARCHIVE"
    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
    if (_numNodes > 1) numberOfNodes = _numNodes
    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
    // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM
    // since we also support multi node integration tests we increase debugPort per node
    if (System.getProperty("cluster.debug") != null) {
        def debugPort = 5005
        nodes.forEach { node ->
            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
            debugPort += 1
        }
    }
    plugin(project.tasks.bundlePlugin.archiveFile)
    plugin(provider(new Callable<RegularFile>(){
        @Override
        RegularFile call() throws Exception {
            return new RegularFile() {
                @Override
                File getAsFile() {
                    return configurations.zipArchive.asFileTree.getSingleFile()
                }
            }
        }
    }))

    nodes.each { node ->
        def plugins = node.plugins
        def firstPlugin = plugins.get(0)
        plugins.remove(0)
        plugins.add(firstPlugin)
    }
}

testClusters.yamlRestTest {

    plugin(provider(new Callable<RegularFile>(){
        @Override
        RegularFile call() throws Exception {
            return new RegularFile() {
                @Override
                File getAsFile() {
                    return configurations.zipArchive.asFileTree.getSingleFile()
                }
            }
        }
    }))
}

task integTestRemote(type: RestIntegTestTask) {
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    // Only rest case can run with remote cluster
    if (System.getProperty("tests.rest.cluster") != null) {
        filter {
            includeTestsMatching "org.opensearch.ml.rest.*IT"
            // mock LLM run in localhost, it will not reachable for docker or remote cluster
            excludeTestsMatching "org.opensearch.ml.tools.VisualizationsToolIT"
        }
    }
}

run {
    doFirst {
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    useCluster testClusters.integTest
}

task release(type: Copy, group: 'build') {
    dependsOn allprojects*.tasks.build
    from(zipTree(project.tasks.bundlePlugin.outputs.files.getSingleFile()))
    into "build/plugins/opensearch-ml"
    includeEmptyDirs = false
}

jacocoTestReport {
    reports {
        xml.getRequired().set(true)
        csv.getRequired().set(false)
        html.getRequired().set(true)
    }

    dependsOn test
}

List<String> jacocoExclusions = [
        // TODO: add more unit test to meet the minimal test coverage.
        'org.opensearch.ml.constant.CommonValue',
        'org.opensearch.ml.plugin.MachineLearningPlugin*',
        'org.opensearch.ml.indices.MLIndicesHandler',
        'org.opensearch.ml.rest.RestMLPredictionAction',
        'org.opensearch.ml.profile.MLModelProfile',
        'org.opensearch.ml.profile.MLPredictRequestStats',
        'org.opensearch.ml.action.deploy.TransportDeployModelAction',
        'org.opensearch.ml.action.deploy.TransportDeployModelOnNodeAction',
        'org.opensearch.ml.action.undeploy.TransportUndeployModelsAction',
        'org.opensearch.ml.action.prediction.TransportPredictionTaskAction',
        'org.opensearch.ml.action.prediction.TransportPredictionTaskAction.1',
        'org.opensearch.ml.action.tasks.GetTaskTransportAction',
        'org.opensearch.ml.action.tasks.SearchTaskTransportAction',
        'org.opensearch.ml.model.MLModelManager',
        'org.opensearch.ml.stats.MLClusterLevelStat',
        'org.opensearch.ml.stats.MLStatLevel',
        'org.opensearch.ml.utils.IndexUtils',
        'org.opensearch.ml.cluster.MLCommonsClusterManagerEventListener',
        'org.opensearch.ml.cluster.DiscoveryNodeHelper.HotDataNodePredicate',
        'org.opensearch.ml.cluster.MLCommonsClusterEventListener',
        'org.opensearch.ml.task.MLTaskManager',
        'org.opensearch.ml.task.MLTrainingTaskRunner',
        'org.opensearch.ml.task.MLPredictTaskRunner',
        'org.opensearch.ml.task.MLTaskDispatcher',
        'org.opensearch.ml.task.MLTrainAndPredictTaskRunner',
        'org.opensearch.ml.task.MLExecuteTaskRunner',
        'org.opensearch.ml.action.profile.MLProfileTransportAction',
        'org.opensearch.ml.rest.RestMLPredictionAction',
        'org.opensearch.ml.breaker.DiskCircuitBreaker',
        'org.opensearch.ml.autoredeploy.MLModelAutoReDeployer.SearchRequestBuilderFactory',
        'org.opensearch.ml.action.training.TrainingITTests',
        'org.opensearch.ml.action.prediction.PredictionITTests',
        'org.opensearch.ml.cluster.MLSyncUpCron',
        'org.opensearch.ml.model.MLModelGroupManager',
        'org.opensearch.ml.helper.ModelAccessControlHelper',
        'org.opensearch.ml.action.models.DeleteModelTransportAction.2',
        'org.opensearch.ml.model.MLModelCacheHelper',
        'org.opensearch.ml.model.MLModelCacheHelper.1',
        'org.opensearch.ml.action.tasks.CancelBatchJobTransportAction',
        'org.opensearch.ml.jobs.MLBatchTaskUpdateExtension',
        'org.opensearch.ml.jobs.MLBatchTaskUpdateJobRunner'

]

jacocoTestCoverageVerification {
    violationRules {
        rule {
            element = 'CLASS'
            excludes = jacocoExclusions
            limit {
                counter = 'BRANCH'
                minimum = 0.7  //TODO: change this value to 0.7
            }
        }
        rule {
            element = 'CLASS'
            excludes = jacocoExclusions
            limit {
                counter = 'LINE'
                value = 'COVEREDRATIO'
                minimum = 0.8  //TODO: change this value to 0.8
            }
        }
    }
    dependsOn jacocoTestReport
}
check.dependsOn jacocoTestCoverageVerification

configurations.all {
    exclude group: "org.jetbrains", module: "annotations"
    exclude group: "com.google.guava", module: "failureaccess"
    resolutionStrategy.force 'org.apache.commons:commons-lang3:3.10'
    resolutionStrategy.force 'commons-logging:commons-logging:1.2'
    resolutionStrategy.force 'org.objenesis:objenesis:3.2'
    resolutionStrategy.force 'net.java.dev.jna:jna:5.11.0'
    resolutionStrategy.force 'org.apache.commons:commons-text:1.10.0'
    resolutionStrategy.force 'com.google.protobuf:protobuf-java:3.25.5'
    resolutionStrategy.force 'org.apache.httpcomponents:httpcore:4.4.15'
    resolutionStrategy.force 'org.apache.httpcomponents:httpclient:4.5.14'
    resolutionStrategy.force 'commons-codec:commons-codec:1.15'
    resolutionStrategy.force 'org.slf4j:slf4j-api:1.7.36'
    resolutionStrategy.force 'org.codehaus.plexus:plexus-utils:3.3.0'
    resolutionStrategy.force 'org.eclipse.platform:org.eclipse.core.runtime:3.29.0'
    resolutionStrategy.force "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
    resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5:5.3.1"
    resolutionStrategy.force "org.apache.httpcomponents.core5:httpcore5-h2:5.3.1"
    resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:5.4.1"
    resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
    resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${versions.jackson_databind}"
    resolutionStrategy.force "jakarta.json:jakarta.json-api:2.1.3"
    resolutionStrategy.force "org.opensearch:opensearch:${opensearch_version}"
    resolutionStrategy.force "org.bouncycastle:bcprov-jdk18on:1.78.1"
}

apply plugin: 'com.netflix.nebula.ospackage'

// This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name
afterEvaluate {
    ospackage {
        packageName = "${name}"
        release = isSnapshot ? "0.1" : '1'
        version = "${project.version}" - "-SNAPSHOT"

        into '/usr/share/opensearch/plugins'
        from(zipTree(bundlePlugin.archivePath)) {
            into opensearchplugin.name
        }

        user 'root'
        permissionGroup 'root'
        fileMode 0644
        dirMode 0755

        requires('opensearch', versions.opensearch, EQUAL)
        packager = 'Amazon'
        vendor = 'Amazon'
        os = 'LINUX'
        prefix '/usr'

        license 'ASL-2.0'
        maintainer 'OpenSearch <opensearch@amazon.com>'
        url 'https://opensearch.org/downloads.html'
        summary '''
         ML plugin for OpenSearch.
         Github https://github.com/opensearch-project/ml-commons.
    '''.stripIndent().replace('\n', ' ').trim()
    }

    buildRpm {
        arch = 'NOARCH'
        dependsOn 'assemble'
    }

    buildDeb {
        arch = 'all'
        dependsOn 'assemble'
    }

    task buildPackages(type: GradleBuild) {
        tasks = ['build', 'buildRpm', 'buildDeb']
    }
}

spotless {
    java {
        removeUnusedImports()
        importOrder 'java', 'javax', 'org', 'com'

        eclipse().withP2Mirrors(Map.of("https://download.eclipse.org/", "https://mirror.umd.edu/eclipse/")).configFile rootProject.file('.eclipseformat.xml')
    }
}

tasks.withType(licenseHeaders.class) {
    additionalLicense 'AL   ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")'
}

String bwcVersion = "2.4.0.0"
String bwcShortVersion = bwcVersion[0..4]
String baseName = "mlCommonsBwcCluster"
String bwcMlPlugin = "opensearch-ml-" + bwcVersion + ".zip"
String bwcFilePath = "src/test/resources/org/opensearch/ml/bwc/"
String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + bwcShortVersion + "/latest/linux/x64/tar/builds/opensearch/plugins/" + bwcMlPlugin
String opensearchMlPlugin = "opensearch-ml-" + project.version + ".zip"

2.times {i ->
    testClusters {
        "${baseName}$i" {
            testDistribution = "ARCHIVE"
            versions = [bwcShortVersion, opensearch_version]
            numberOfNodes = 3
            plugin(provider(new Callable<RegularFile>() {
                @Override
                RegularFile call() throws Exception {
                    return new RegularFile() {
                        @Override
                        File getAsFile() {
                            File bwcDir = new File('./plugin/' + bwcFilePath)
                            if (!bwcDir.exists()) {
                                bwcDir.mkdirs()
                            }
                            File dir = new File('./plugin/' + bwcFilePath + bwcVersion)
                            if (!dir.exists()) {
                                dir.mkdirs()
                            }
                            File f = new File(dir, bwcMlPlugin)
                            if (!f.exists()) {
                                new URL(bwcRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
                            }
                            return fileTree(bwcFilePath + bwcVersion).getSingleFile()
                        }
                    }
                }
            }))
            setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
            setting 'http.content_type.required', 'true'
        }
    }
}

List<Provider<RegularFile>> plugins = [
        provider(new Callable<RegularFile>() {
            @Override
            RegularFile call() throws Exception {
                return new RegularFile() {
                    @Override
                    File getAsFile() {
                        project.mkdir "$bwcFilePath/$project.version"
                        copy {
                            from "$buildDir/distributions/$opensearchMlPlugin"
                            into "$bwcFilePath/$project.version"
                        }
                        return fileTree(bwcFilePath + project.version).getSingleFile()
                    }
                }
            }
        })
]

// Creates 2 test clusters with 3 nodes of the old version.
2.times { i ->
    task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
        useCluster testClusters."${baseName}$i"
        filter {
            includeTestsMatching "org.opensearch.ml.bwc.*IT"
        }
        systemProperty 'tests.rest.bwcsuite', 'old_cluster'
        systemProperty 'tests.rest.bwcsuite_round', 'old'
        systemProperty 'tests.plugin_bwc_version', bwcVersion
        nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
        nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
    }
}

// Upgrade one node of the old cluster to new OpenSearch version with upgraded plugin version
// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
// This is also used as a one third upgraded cluster for a rolling upgrade.
task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
    useCluster testClusters."${baseName}0"
    dependsOn "${baseName}#oldVersionClusterTask0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.ml.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'first'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
// This is used for rolling upgrade.
task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#mixedClusterTask"
    useCluster testClusters."${baseName}0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.ml.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'second'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrade the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
// This results in a fully upgraded cluster.
// This is used for rolling upgrade.
task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
    useCluster testClusters."${baseName}0"
    doFirst {
        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.ml.bwc.*IT"
    }
    mustRunAfter "${baseName}#mixedClusterTask"
    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
    systemProperty 'tests.rest.bwcsuite_round', 'third'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
// at the same time resulting in a fully upgraded cluster.
task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
    dependsOn "${baseName}#oldVersionClusterTask1"
    useCluster testClusters."${baseName}1"
    doFirst {
        testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
    }
    filter {
        includeTestsMatching "org.opensearch.ml.bwc.*IT"
    }
    systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
    systemProperty 'tests.plugin_bwc_version', bwcVersion
    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
}

// A bwc test suite which runs all the bwc tasks combined
task bwcTestSuite(type: StandaloneRestIntegTestTask) {
    exclude '**/*Test*'
    exclude '**/*IT*'
    dependsOn tasks.named("${baseName}#mixedClusterTask")
    dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
    dependsOn tasks.named("${baseName}#fullRestartClusterTask")
}

forbiddenPatterns {
    exclude '**/*.pdf'
    exclude '**/*.jpg'
}