Merge branch 'main' into metadata2

Signed-off-by: Muneer Kolarkunnu <[email protected]>

Author: akolarkunnu

.github/workflows/assemble.yml

@@ -7,35 +7,21 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
-        os: [ubuntu-latest, windows-latest, macos-13]
+        java: [ 21, 23 ]
+        os: [ubuntu-latest, windows-latest, macos-13, ubuntu-24.04-arm]
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
         uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
           distribution: temurin
-      - name: Set up JDK 17
-        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
-        if: matrix.java == 11
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Set JAVA${{ matrix.java }}_HOME
-        shell: bash
-        run: |
-          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Setup docker (missing on MacOS)
         id: setup_docker
         if: runner.os == 'macos'
         continue-on-error: true
         run: |
-          # Force QEMU 9.0.2 usage
-          curl https://raw.githubusercontent.com/Homebrew/homebrew-core/f1a9cf104a9a51779c7a532b658c490f69974839/Formula/q/qemu.rb > qemu.rb
-          brew install qemu.rb
-          HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 HOMEBREW_NO_AUTO_UPDATE=1 brew install docker colima coreutils
+          brew install docker colima coreutils
           gtimeout 15m colima start
         shell: bash
       - name: Run Gradle (assemble)
@@ -47,8 +33,8 @@ jobs:
         shell: bash
         if: runner.os != 'macos'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }} 
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE 
       - name: Run Gradle (assemble)
         if: runner.os == 'macos' && steps.setup_docker.outcome == 'success'
         run: |
-          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }}
+          ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE

.github/workflows/precommit.yml

@@ -7,8 +7,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        java: [ 11, 17, 21, 23 ]
-        os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
+        java: [ 21, 23 ]
+        os: [ubuntu-latest, windows-latest, macos-latest, macos-13, ubuntu-24.04-arm]
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
@@ -17,18 +17,7 @@ jobs:
           java-version: ${{ matrix.java }}
           distribution: temurin
           cache: gradle
-      - name: Set up JDK 17
-        # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
-        if: matrix.java == 11
-        uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Set JAVA${{ matrix.java }}_HOME
-        shell: bash
-        run: |
-          echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
       - name: Run Gradle (precommit)
         shell: bash
         run: |
-          ./gradlew javadoc precommit --parallel -Druntime.java=${{ matrix.java }}
+          ./gradlew javadoc precommit --parallel

.github/workflows/publish-maven-snapshots.yml

@@ -19,11 +19,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - name: Set up JDK 17
+      - name: Set up JDK 21
         uses: actions/setup-java@v4
         with:
-          distribution: adopt
-          java-version: 17
+          distribution: temurin
+          java-version: 21
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
@@ -37,4 +37,4 @@ jobs:
           export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
           echo "::add-mask::$SONATYPE_USERNAME"
           echo "::add-mask::$SONATYPE_PASSWORD"
-          ./gradlew publishNebulaPublicationToSnapshotsRepository
+          ./gradlew publishNebulaPublicationToSnapshotsRepository -Dbuild.version_qualifier=alpha1

CHANGELOG-3.0.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
 
 ### Dependencies
+- Update Apache Lucene to 10.1.0 ([#16366](https://github.com/opensearch-project/OpenSearch/pull/16366))
 - Bump Apache HttpCore5/HttpClient5 dependencies from 5.2.5/5.3.1 to 5.3.1/5.4.1 to support ExtendedSocketOption in HttpAsyncClient ([#16757](https://github.com/opensearch-project/OpenSearch/pull/16757))
 
 ### Changed
@@ -26,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add task completion count in search backpressure stats API ([#10028](https://github.com/opensearch-project/OpenSearch/pull/10028/))
 - Deprecate CamelCase `PathHierarchy` tokenizer name in favor to lowercase `path_hierarchy` ([#10894](https://github.com/opensearch-project/OpenSearch/pull/10894))
 - Breaking change: Do not request "search_pipelines" metrics by default in NodesInfoRequest ([#12497](https://github.com/opensearch-project/OpenSearch/pull/12497))
+- Refactor `:libs` module `bootstrap` package to eliminate top level split packages [#17117](https://github.com/opensearch-project/OpenSearch/pull/17117))
 - Breaking change: Modify the utility APIs in the Metadata to get different indices ([#14723](https://github.com/opensearch-project/OpenSearch/pull/14723))
 
 ### Deprecated
@@ -46,6 +48,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021))
 - Remove custom Map, List and Set collection classes ([#6871](https://github.com/opensearch-project/OpenSearch/pull/6871))
 - Remove `index.store.hybrid.mmap.extensions` setting in favor of `index.store.hybrid.nio.extensions` setting ([#9392](https://github.com/opensearch-project/OpenSearch/pull/9392))
+- Move o.o.action.support.master classes ([#17104](https://github.com/opensearch-project/OpenSearch/pull/17104))
 
 ### Fixed
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))

CHANGELOG.md

@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add stats for remote publication failure and move download failure stats to remote methods([#16682](https://github.com/opensearch-project/OpenSearch/pull/16682/))
 - Update script supports java.lang.String.sha1() and java.lang.String.sha256() methods ([#16923](https://github.com/opensearch-project/OpenSearch/pull/16923))
 - Added a precaution to handle extreme date values during sorting to prevent `arithmetic_exception: long overflow` ([#16812](https://github.com/opensearch-project/OpenSearch/pull/16812)).
+- Add `verbose_pipeline` parameter to output each processor's execution details ([#16843](https://github.com/opensearch-project/OpenSearch/pull/16843)).
 - Add search replica stats to segment replication stats API ([#16678](https://github.com/opensearch-project/OpenSearch/pull/16678))
 - Introduce a setting to disable download of full cluster state from remote on term mismatch([#16798](https://github.com/opensearch-project/OpenSearch/pull/16798/))
 - Added ability to retrieve value from DocValues in a flat_object filed([#16802](https://github.com/opensearch-project/OpenSearch/pull/16802))
@@ -32,6 +33,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support object fields in star-tree index([#16728](https://github.com/opensearch-project/OpenSearch/pull/16728/))
 - Support searching from doc_value using termQueryCaseInsensitive/termQuery in flat_object/keyword field([#16974](https://github.com/opensearch-project/OpenSearch/pull/16974/))
 - Added a new `time` field to replace the deprecated `getTime` field in `GetStats`. ([#17009](https://github.com/opensearch-project/OpenSearch/pull/17009))
+- Improve flat_object field parsing performance by reducing two passes to a single pass ([#16297](https://github.com/opensearch-project/OpenSearch/pull/16297))
+- Improve performance of the bitmap filtering([#16936](https://github.com/opensearch-project/OpenSearch/pull/16936/))
+- Added new Setting property UnmodifiableOnRestore to prevent updating settings on restore snapshot ([#16957](https://github.com/opensearch-project/OpenSearch/pull/16957))
+- Introduce Template query ([#16818](https://github.com/opensearch-project/OpenSearch/pull/16818))
+- Propagate the sourceIncludes and excludes fields from fetchSourceContext to FieldsVisitor. ([#17080](https://github.com/opensearch-project/OpenSearch/pull/17080))
 
 ### Dependencies
 - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))
@@ -66,6 +72,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `opentelemetry-semconv` from 1.27.0-alpha to 1.29.0-alpha ([#16700](https://github.com/opensearch-project/OpenSearch/pull/16700))
 - Bump `com.google.re2j:re2j` from 1.7 to 1.8 ([#17012](https://github.com/opensearch-project/OpenSearch/pull/17012))
 - Bump `com.squareup.okio:okio` from 3.9.1 to 3.10.2 ([#17060](https://github.com/opensearch-project/OpenSearch/pull/17060))
+- Bump `org.jruby.jcodings:jcodings` from 1.0.58 to 1.0.61 ([#17061](https://github.com/opensearch-project/OpenSearch/pull/17061))
 
 ### Changed
 - Indexed IP field supports `terms_query` with more than 1025 IP masks [#16391](https://github.com/opensearch-project/OpenSearch/pull/16391)
@@ -74,6 +81,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Allow extended plugins to be optional ([#16909](https://github.com/opensearch-project/OpenSearch/pull/16909))
 - Use the correct type to widen the sort fields when merging top docs ([#16881](https://github.com/opensearch-project/OpenSearch/pull/16881))
 - Limit reader writer separation to remote store enabled clusters [#16760](https://github.com/opensearch-project/OpenSearch/pull/16760)
+- Optimize innerhits query performance [#16937](https://github.com/opensearch-project/OpenSearch/pull/16937)
 
 ### Deprecated
 - Performing update operation with default pipeline or final pipeline is deprecated ([#16712](https://github.com/opensearch-project/OpenSearch/pull/16712))
@@ -103,7 +111,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix Shallow copy snapshot failures on closed index ([#16868](https://github.com/opensearch-project/OpenSearch/pull/16868))
 - Fix multi-value sort for unsigned long ([#16732](https://github.com/opensearch-project/OpenSearch/pull/16732))
 - The `phone-search` analyzer no longer emits the tel/sip prefix, international calling code, extension numbers and unformatted input as a token ([#16993](https://github.com/opensearch-project/OpenSearch/pull/16993))
+- Stop processing search requests when _msearch request is cancelled ([#17005](https://github.com/opensearch-project/OpenSearch/pull/17005))
 - Fix GRPC AUX_TRANSPORT_PORT and SETTING_GRPC_PORT settings and remove lingering HTTP terminology ([#17037](https://github.com/opensearch-project/OpenSearch/pull/17037))
+- Fix exists queries on nested flat_object fields throws exception ([#16803](https://github.com/opensearch-project/OpenSearch/pull/16803))
+- Use OpenSearch version to deserialize remote custom metadata([#16494](https://github.com/opensearch-project/OpenSearch/pull/16494))
 
 ### Security
 

MAINTAINERS.md

@@ -16,7 +16,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Craig Perkins            | [cwperks](https://github.com/cwperks)                   | Amazon      |
 | Dan Widdis               | [dbwiddis](https://github.com/dbwiddis)                 | Amazon      |
 | Daniel "dB." Doubrovkine | [dblock](https://github.com/dblock)                     | Amazon      |
-| Gao Binlong              | [gaobinlong](https://github.com/gaobinlong)             | Amazon      |
+| Binlong Gao              | [gaobinlong](https://github.com/gaobinlong)             | Amazon      |
 | Gaurav Bafna             | [gbbafna](https://github.com/gbbafna)                   | Amazon      |
 | Jay Deng                 | [jed326](https://github.com/jed326)                     | Amazon      |
 | Kunal Kotwani            | [kotwanikunal](https://github.com/kotwanikunal)         | Amazon      |

benchmarks/build.gradle

@@ -85,44 +85,5 @@ spotless {
   }
 }
 
-if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_20) {
-  // Add support for incubator modules on supported Java versions.
-  run.jvmArgs += ['--add-modules=jdk.incubator.vector']
-  run.classpath += files(jar.archiveFile)
-  run.classpath -= sourceSets.main.output
-  evaluationDependsOn(':libs:opensearch-common')
-
-  sourceSets {
-    java20 {
-      java {
-        srcDirs = ['src/main/java20']
-      }
-    }
-  }
-
-  configurations {
-    java20Implementation.extendsFrom(implementation)
-  }
-
-  dependencies {
-    java20Implementation sourceSets.main.output
-    java20Implementation project(':libs:opensearch-common').sourceSets.java20.output
-    java20AnnotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
-  }
-
-  compileJava20Java {
-    targetCompatibility = JavaVersion.VERSION_20
-    options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])
-  }
-
-  jar {
-    metaInf {
-      into 'versions/20'
-      from sourceSets.java20.output
-    }
-    manifest.attributes('Multi-Release': 'true')
-  }
-
-  // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
-  disableTasks('forbiddenApisJava20')
-}
+// Add support for incubator modules on supported Java versions.
+run.jvmArgs += ['--add-modules=jdk.incubator.vector']

benchmarks/src/main/java/org/opensearch/common/round/RoundableSupplier.java

@@ -13,7 +13,7 @@
 public class RoundableSupplier implements Supplier<Roundable> {
     private final Supplier<Roundable> delegate;
 
-    RoundableSupplier(String type, long[] values, int size) throws ClassNotFoundException {
+    RoundableSupplier(String type, long[] values, int size) {
         switch (type) {
             case "binary":
                 delegate = () -> new BinarySearcher(values, size);
@@ -22,7 +22,8 @@ public class RoundableSupplier implements Supplier<Roundable> {
                 delegate = () -> new BidirectionalLinearSearcher(values, size);
                 break;
             case "btree":
-                throw new ClassNotFoundException("BtreeSearcher is not supported below JDK 20");
+                delegate = () -> new BtreeSearcher(values, size);
+                break;
             default:
                 throw new IllegalArgumentException("invalid type: " + type);
         }

buildSrc/build.gradle

@@ -74,8 +74,8 @@ processResources {
  *         Java version                                                      *
  *****************************************************************************/
 
-if (JavaVersion.current() < JavaVersion.VERSION_11) {
-  throw new GradleException('At least Java 11 is required to build opensearch gradle tools')
+if (JavaVersion.current() < JavaVersion.VERSION_21) {
+  throw new GradleException('At least Java 21 is required to build opensearch gradle tools')
 }
 
 sourceSets {
@@ -111,7 +111,7 @@ dependencies {
   api 'org.apache.rat:apache-rat:0.15'
   api "commons-io:commons-io:${props.getProperty('commonsio')}"
   api "net.java.dev.jna:jna:5.14.0"
-  api 'com.github.johnrengelman:shadow:8.1.1'
+  api 'com.gradleup.shadow:shadow-gradle-plugin:8.3.5'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
   api 'de.thetaphi:forbiddenapis:3.8'
@@ -171,8 +171,8 @@ if (project != rootProject) {
 
   allprojects {
     java {
-        targetCompatibility = JavaVersion.VERSION_11
-        sourceCompatibility = JavaVersion.VERSION_11
+        targetCompatibility = JavaVersion.VERSION_21
+        sourceCompatibility = JavaVersion.VERSION_21
     }
   }
 

buildSrc/reaper/build.gradle

@@ -12,8 +12,8 @@
 apply plugin: 'java'
 
 java {
-  targetCompatibility = JavaVersion.VERSION_11
-  sourceCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
+  sourceCompatibility = JavaVersion.VERSION_21
 }
 
 jar {

buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/.ci/java-versions.properties

@@ -27,6 +27,6 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-OPENSEARCH_BUILD_JAVA=openjdk11
-OPENSEARCH_RUNTIME_JAVA=java11
+OPENSEARCH_BUILD_JAVA=openjdk21
+OPENSEARCH_RUNTIME_JAVA=java21
 GRADLE_TASK=build

buildSrc/src/main/java/org/opensearch/gradle/precommit/JarHellTask.java

@@ -63,7 +63,7 @@ public JarHellTask(Project project) {
     public void runJarHellCheck() {
         LoggedExec.javaexec(project, spec -> {
             spec.environment("CLASSPATH", getClasspath().getAsPath());
-            spec.getMainClass().set("org.opensearch.bootstrap.JarHell");
+            spec.getMainClass().set("org.opensearch.common.bootstrap.JarHell");
         });
     }
 

buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -94,7 +94,7 @@ public class ThirdPartyAuditTask extends DefaultTask {
         CliMain.EXIT_VIOLATION,
         CliMain.EXIT_UNSUPPORTED_JDK
     );
-    private static final String JDK_JAR_HELL_MAIN_CLASS = "org.opensearch.bootstrap.JdkJarHellCheck";
+    private static final String JDK_JAR_HELL_MAIN_CLASS = "org.opensearch.common.bootstrap.JdkJarHellCheck";
 
     private Set<String> missingClassExcludes = new TreeSet<>();
 

buildSrc/src/main/resources/forbidden/opensearch-server-signatures.txt

@@ -61,10 +61,6 @@ java.nio.channels.FileChannel#read(java.nio.ByteBuffer, long)
 @defaultMessage Use Lucene.parseLenient instead it strips off minor version
 org.apache.lucene.util.Version#parseLeniently(java.lang.String)
 
-@defaultMessage Spawns a new thread which is solely under lucenes control use ThreadPool#relativeTimeInMillis instead
-org.apache.lucene.search.TimeLimitingCollector#getGlobalTimerThread()
-org.apache.lucene.search.TimeLimitingCollector#getGlobalCounter()
-
 @defaultMessage Don't interrupt threads use FutureUtils#cancel(Future<T>) instead
 java.util.concurrent.Future#cancel(boolean)
 

buildSrc/src/main/resources/minimumCompilerVersion

@@ -1 +1 @@
-17
+21

buildSrc/src/main/resources/minimumRuntimeVersion

@@ -1 +1 @@
-11
+21

buildSrc/src/testFixtures/java/org/opensearch/gradle/test/TestClasspathUtils.java

@@ -48,7 +48,7 @@ public class TestClasspathUtils {
     public static void setupJarJdkClasspath(File projectRoot) {
         try {
             URL originLocation = TestClasspathUtils.class.getClassLoader()
-                .loadClass("org.opensearch.bootstrap.JdkJarHellCheck")
+                .loadClass("org.opensearch.common.bootstrap.JdkJarHellCheck")
                 .getProtectionDomain()
                 .getCodeSource()
                 .getLocation();

buildSrc/src/testKit/thirdPartyAudit/build.gradle

@@ -46,11 +46,11 @@ dependencies {
 }
 
 tasks.register("empty", ThirdPartyAuditTask) {
-  targetCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
   signatureFile = file('third-party-audit-empty.txt')
 }
 
 tasks.register("absurd", ThirdPartyAuditTask) {
-  targetCompatibility = JavaVersion.VERSION_11
+  targetCompatibility = JavaVersion.VERSION_21
   signatureFile = file('third-party-audit-absurd.txt')
 }

client/rest-high-level/src/main/java/org/opensearch/client/ClusterClient.java

@@ -38,7 +38,7 @@
 import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsResponse;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
-import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.action.support.clustermanager.AcknowledgedResponse;
 import org.opensearch.client.cluster.RemoteInfoRequest;
 import org.opensearch.client.cluster.RemoteInfoResponse;
 import org.opensearch.client.indices.ComponentTemplatesExistRequest;