Make LockService an interface and replace usages of ThreadContext.stashContext

build.gradle

@@ -57,21 +57,6 @@ ext {
     projectSubstitutions = [:]
     licenseFile = rootProject.file('LICENSE.txt')
     noticeFile = rootProject.file('NOTICE')
-
-    runIntegTestWithSecurityPlugin= System.getProperty("security.enabled")
-    if (runIntegTestWithSecurityPlugin == "true"){
-        ['esnode.pem', 'esnode-key.pem', 'kirk.pem', 'kirk-key.pem', 'root-ca.pem', 'sample.pem', 'test-kirk.jks'].forEach { file ->
-            File local = getLayout().getBuildDirectory().file(file).get().getAsFile()
-            download.run {
-                src "https://raw.githubusercontent.com/opensearch-project/security/refs/heads/main/bwc-test/src/test/resources/security/" + file
-                dest local
-                overwrite false
-            }
-            processResources {
-                from(local)
-            }
-        }
-    }
 }
 
 licenseHeaders.enabled = true
@@ -110,6 +95,82 @@ allprojects {
         targetCompatibility = JavaVersion.VERSION_21
         sourceCompatibility = JavaVersion.VERSION_21
     }
+
+    // Make security settings available to all subprojects
+    ext.securityEnabled = System.getProperty("security.enabled", "false") == "true" || System.getProperty("https", "false") == "true"
+
+    // Download security files for projects that need them
+    if (securityEnabled && (project == rootProject || project.name == 'opensearch-job-scheduler-sample-extension')) {
+        afterEvaluate {
+            ['esnode.pem', 'esnode-key.pem', 'kirk.pem', 'kirk-key.pem', 'root-ca.pem', 'sample.pem', 'test-kirk.jks'].forEach { file ->
+                File local = project.layout.buildDirectory.file(file).get().asFile
+                download.run {
+                    src "https://raw.githubusercontent.com/opensearch-project/security/refs/heads/main/bwc-test/src/test/resources/security/" + file
+                    dest local
+                    overwrite false
+                }
+                processTestResources {
+                    from(local)
+                }
+            }
+        }
+    }
+    ext.resolvePluginFile = { pluginId ->
+        return new Callable<RegularFile>() {
+            @Override
+            RegularFile call() throws Exception {
+                return new RegularFile() {
+                    @Override
+                    File getAsFile() {
+                        return configurations.opensearchPlugin.resolvedConfiguration.resolvedArtifacts
+                                .find { ResolvedArtifact f ->
+                                    f.name.startsWith(pluginId)
+                                }
+                                .file
+                    }
+                }
+            }
+        }
+    }
+    ext.securityPluginFile = resolvePluginFile("opensearch-security")
+    afterEvaluate {
+        if (plugins.hasPlugin('opensearch.testclusters') && configurations.findByName('opensearchPlugin')) {
+            // === Setup security test ===
+            testClusters.integTest.nodes.each { node ->
+                def plugins = node.plugins
+                def firstPlugin = plugins.get(0)
+                if (firstPlugin.provider == project.bundlePlugin.archiveFile) {
+                    plugins.remove(0)
+                    plugins.add(firstPlugin)
+                }
+
+                if (ext.securityEnabled) {
+                    node.extraConfigFile("kirk.pem", file("build/kirk.pem"))
+                    node.extraConfigFile("kirk-key.pem", file("build/kirk-key.pem"))
+                    node.extraConfigFile("esnode.pem", file("build/esnode.pem"))
+                    node.extraConfigFile("esnode-key.pem", file("build/esnode-key.pem"))
+                    node.extraConfigFile("root-ca.pem", file("build/root-ca.pem"))
+                    node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
+                    node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
+                    node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
+                    node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
+                    node.setting("plugins.security.ssl.http.enabled", "true")
+                    node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
+                    node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
+                    node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
+                    node.setting("plugins.security.allow_unsafe_democertificates", "true")
+                    node.setting("plugins.security.allow_default_init_securityindex", "true")
+                    node.setting("plugins.security.authcz.admin_dn", "\n - CN=kirk,OU=client,O=client,L=test,C=de")
+                    node.setting("plugins.security.audit.type", "internal_opensearch")
+                    node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
+                    node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
+                    node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
+                    node.setting("plugins.security.system_indices.enabled", "true")
+                    // node.setting("plugins.security.system_indices.indices", "[\".opendistro-ism-config\"]")
+                }
+            }
+        }
+    }
 }
 
 allprojects {
@@ -237,66 +298,6 @@ integTest.getClusters().forEach{c -> {
     c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))
 }}
 
-ext.resolvePluginFile = { pluginId ->
-    return new Callable<RegularFile>() {
-        @Override
-        RegularFile call() throws Exception {
-            return new RegularFile() {
-                @Override
-                File getAsFile() {
-                    return configurations.opensearchPlugin.resolvedConfiguration.resolvedArtifacts
-                            .find { ResolvedArtifact f ->
-                                f.name.startsWith(pluginId)
-                            }
-                            .file
-                }
-            }
-        }
-    }
-}
-def securityPluginFile = resolvePluginFile("opensearch-security")
-
-// === Setup security test ===
-// This flag indicates the existence of security plugin
-def securityEnabled = System.getProperty("security.enabled", "false") == "true" || System.getProperty("https", "false") == "true"
-afterEvaluate {
-    testClusters.integTest.nodes.each { node ->
-        def plugins = node.plugins
-        def firstPlugin = plugins.get(0)
-        if (firstPlugin.provider == project.bundlePlugin.archiveFile) {
-            plugins.remove(0)
-            plugins.add(firstPlugin)
-        }
-
-        if (securityEnabled) {
-            node.extraConfigFile("kirk.pem", file("build/resources/main/kirk.pem"))
-            node.extraConfigFile("kirk-key.pem", file("build/resources/main/kirk-key.pem"))
-            node.extraConfigFile("esnode.pem", file("build/resources/main/esnode.pem"))
-            node.extraConfigFile("esnode-key.pem", file("build/resources/main/esnode-key.pem"))
-            node.extraConfigFile("root-ca.pem", file("build/resources/main/root-ca.pem"))
-            node.setting("network.bind_host", "127.0.0.1")
-            node.setting("network.publish_host", "127.0.0.1")
-            node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
-            node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
-            node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
-            node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
-            node.setting("plugins.security.ssl.http.enabled", "true")
-            node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
-            node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
-            node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
-            node.setting("plugins.security.allow_unsafe_democertificates", "true")
-            node.setting("plugins.security.allow_default_init_securityindex", "true")
-            node.setting("plugins.security.authcz.admin_dn", "\n - CN=kirk,OU=client,O=client,L=test,C=de")
-            node.setting("plugins.security.audit.type", "internal_opensearch")
-            node.setting("plugins.security.enable_snapshot_restore_privilege", "true")
-            node.setting("plugins.security.check_snapshot_restore_write_privileges", "true")
-            node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]")
-            node.setting("plugins.security.system_indices.enabled", "true")
-            // node.setting("plugins.security.system_indices.indices", "[\".opendistro-ism-config\"]")
-        }
-    }
-}
-
 testClusters.integTest {
     testDistribution = 'INTEG_TEST'
 
@@ -313,8 +314,8 @@ testClusters.integTest {
         }
     }
 
-    if (securityEnabled) {
-        plugin(provider(securityPluginFile))
+    if (ext.securityEnabled) {
+        plugin(provider(ext.securityPluginFile))
     }
     setting 'path.repo', repo.absolutePath
 }

sample-extension-plugin/build.gradle

@@ -42,6 +42,10 @@ repositories {
     maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
 }
 
+configurations {
+    opensearchPlugin
+}
+
 dependencies {
     compileOnly project(path: ":${rootProject.name}-spi", configuration: 'shadow')
     testImplementation('org.awaitility:awaitility:4.3.0') {
@@ -127,8 +131,12 @@ testClusters.integTest {
             debugPort += 1
         }
     }
+    if (ext.securityEnabled) {
+        plugin(provider(ext.securityPluginFile))
+    }
     setting 'path.repo', repo.absolutePath
     setting 'plugins.jobscheduler.sweeper.period', '1s'
+    setting 'plugins.jobscheduler.history.enabled', 'true'
 }
 
 String baseName = "jobSchedulerBwcCluster"

sample-extension-plugin/src/test/java/org/opensearch/jobscheduler/sampleextension/SampleExtensionIntegTestCase.java

@@ -23,6 +23,7 @@
 import org.apache.hc.core5.reactor.ssl.TlsDetails;
 import org.apache.hc.core5.ssl.SSLContextBuilder;
 import org.apache.hc.core5.util.Timeout;
+import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.opensearch.client.Request;
@@ -36,8 +37,11 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
+import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.jobscheduler.spi.LockModel;
 import org.opensearch.jobscheduler.spi.schedule.CronSchedule;
 import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule;
@@ -77,6 +81,56 @@
 
 public class SampleExtensionIntegTestCase extends OpenSearchRestTestCase {
 
+    @After
+    public void wipeAllODFEIndices() throws IOException {
+        Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all"));
+
+        MediaType mediaType = MediaType.fromMediaType(response.getEntity().getContentType());
+
+        try (
+            XContentParser parser = mediaType.xContent()
+                .createParser(
+                    NamedXContentRegistry.EMPTY,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                    response.getEntity().getContent()
+                )
+        ) {
+
+            XContentParser.Token token = parser.nextToken();
+            if (token != XContentParser.Token.START_ARRAY) {
+                throw new IOException("Expected START_ARRAY from /_cat/indices response but found: " + token);
+            }
+
+            // Iterate over array elements (each is an object with "index" and other fields)
+            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                String indexName = null;
+
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        String field = parser.currentName();
+                        token = parser.nextToken();
+                        if ("index".equals(field)) {
+                            indexName = parser.text();
+                        } else {
+                            // skip any nested structures we don't care about
+                            parser.skipChildren();
+                        }
+                    }
+                }
+
+                // .opendistro_security isn't allowed to delete from cluster
+                if (indexName != null && !".opendistro_security".equals(indexName)) {
+                    Request delete = new Request("DELETE", "/" + indexName);
+                    RequestOptions.Builder opts = RequestOptions.DEFAULT.toBuilder();
+                    opts.setWarningsHandler(WarningsHandler.PERMISSIVE);
+                    delete.setOptions(opts.build());
+
+                    adminClient().performRequest(delete);
+                }
+            }
+        }
+    }
+
     @AfterClass
     public static void dumpCoverage() throws IOException, MalformedObjectNameException {
         // jacoco.dir is set in esplugin-coverage.gradle, if it doesn't exist we don't
@@ -186,7 +240,6 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s
         String userName = Optional.ofNullable(System.getProperty("user")).orElseThrow(() -> new RuntimeException("user name is missing"));
         String password = Optional.ofNullable(System.getProperty("password"))
             .orElseThrow(() -> new RuntimeException("password is missing"));
-
         headers.put(
             "Authorization",
             "Basic " + Base64.getEncoder().encodeToString((userName + ":" + password).getBytes(StandardCharsets.UTF_8))
@@ -246,7 +299,7 @@ protected SampleJobParameter createWatcherJob(String jobId, SampleJobParameter j
             LoggingDeprecationHandler.INSTANCE,
             response.getEntity().getContent()
         ).map();
-        return getJobParameter(client(), responseJson.get("_id").toString());
+        return getJobParameter(responseJson.get("_id").toString());
     }
 
     protected void deleteWatcherJob(String jobId) throws IOException {
@@ -272,7 +325,7 @@ protected SampleJobParameter disableWatcherJob(String jobId, SampleJobParameter
             LoggingDeprecationHandler.INSTANCE,
             response.getEntity().getContent()
         ).map();
-        return getJobParameter(client(), responseJson.get("_id").toString());
+        return getJobParameter(responseJson.get("_id").toString());
     }
 
     protected SampleJobParameter enableWatcherJob(String jobId, SampleJobParameter jobParameter) throws IOException {
@@ -286,7 +339,7 @@ protected SampleJobParameter enableWatcherJob(String jobId, SampleJobParameter j
             LoggingDeprecationHandler.INSTANCE,
             response.getEntity().getContent()
         ).map();
-        return getJobParameter(client(), responseJson.get("_id").toString());
+        return getJobParameter(responseJson.get("_id").toString());
     }
 
     protected Response makeRequest(
@@ -328,9 +381,9 @@ protected Map<String, String> getJobParameterAsMap(String jobId, SampleJobParame
     }
 
     @SuppressWarnings("unchecked")
-    protected SampleJobParameter getJobParameter(RestClient client, String jobId) throws IOException {
+    protected SampleJobParameter getJobParameter(String jobId) throws IOException {
         Request request = new Request("POST", "/" + SampleExtensionPlugin.JOB_INDEX_NAME + "/_search");
-        String entity = """
+        String entity = String.format(Locale.ROOT, """
             {
                 "query": {
                     "match": {
@@ -340,9 +393,9 @@ protected SampleJobParameter getJobParameter(RestClient client, String jobId) th
                     }
                 }
             }
-            """.formatted(jobId);
+            """, jobId);
         request.setJsonEntity(entity);
-        Response response = client.performRequest(request);
+        Response response = adminClient().performRequest(request);
         Map<String, Object> responseJson = JsonXContent.jsonXContent.createParser(
             NamedXContentRegistry.EMPTY,
             LoggingDeprecationHandler.INSTANCE,
@@ -441,7 +494,7 @@ protected void waitUntilLockIsAcquiredAndReleased(String jobId, int maxTimeInSec
 
     @SuppressWarnings("unchecked")
     protected LockModel getLockByJobId(String jobId) throws IOException {
-        String entity = """
+        String entity = String.format(Locale.ROOT, """
             {
                 "query": {
                     "match": {
@@ -451,9 +504,9 @@ protected LockModel getLockByJobId(String jobId) throws IOException {
                     }
                 }
             }
-            """.formatted(jobId);
+            """, jobId);
         Response response = makeRequest(
-            client(),
+            adminClient(),
             "POST",
             "/.opendistro-job-scheduler-lock/_search",
             Map.of("ignore", "404"),