Merge branch 'main' into template-version

Author: davidkyle

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -26,7 +26,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
@@ -227,50 +226,44 @@ private static BlockLoader blockLoader(String name) {
                     break;
             }
             ft.freeze();
-            return new KeywordFieldMapper.KeywordFieldType(
-                w.name,
-                ft,
-                Lucene.KEYWORD_ANALYZER,
-                Lucene.KEYWORD_ANALYZER,
-                Lucene.KEYWORD_ANALYZER,
-                new KeywordFieldMapper.Builder(name, defaultIndexSettings()).docValues(ft.docValuesType() != DocValuesType.NONE),
-                syntheticSource
-            ).blockLoader(new MappedFieldType.BlockLoaderContext() {
-                @Override
-                public String indexName() {
-                    return "benchmark";
-                }
+            return new KeywordFieldMapper.KeywordFieldType(w.name, ft, syntheticSource).blockLoader(
+                new MappedFieldType.BlockLoaderContext() {
+                    @Override
+                    public String indexName() {
+                        return "benchmark";
+                    }
 
-                @Override
-                public IndexSettings indexSettings() {
-                    throw new UnsupportedOperationException();
-                }
+                    @Override
+                    public IndexSettings indexSettings() {
+                        throw new UnsupportedOperationException();
+                    }
 
-                @Override
-                public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
-                    return MappedFieldType.FieldExtractPreference.NONE;
-                }
+                    @Override
+                    public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
+                        return MappedFieldType.FieldExtractPreference.NONE;
+                    }
 
-                @Override
-                public SearchLookup lookup() {
-                    throw new UnsupportedOperationException();
-                }
+                    @Override
+                    public SearchLookup lookup() {
+                        throw new UnsupportedOperationException();
+                    }
 
-                @Override
-                public Set<String> sourcePaths(String name) {
-                    return Set.of(name);
-                }
+                    @Override
+                    public Set<String> sourcePaths(String name) {
+                        return Set.of(name);
+                    }
 
-                @Override
-                public String parentField(String field) {
-                    throw new UnsupportedOperationException();
-                }
+                    @Override
+                    public String parentField(String field) {
+                        throw new UnsupportedOperationException();
+                    }
 
-                @Override
-                public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
-                    return FieldNamesFieldMapper.FieldNamesFieldType.get(true);
+                    @Override
+                    public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
+                        return FieldNamesFieldMapper.FieldNamesFieldType.get(true);
+                    }
                 }
-            });
+            );
         }
         throw new IllegalArgumentException("can't read [" + name + "]");
     }

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -217,7 +217,12 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 );
                 yield EvalMapper.toEvaluator(
                     FOLD_CONTEXT,
-                    new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION), timestamp),
+                    new DateTrunc(
+                        Source.EMPTY,
+                        new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION),
+                        timestamp,
+                        configuration()
+                    ),
                     layout(timestamp)
                 ).get(driverContext);
             }

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java

@@ -18,7 +18,9 @@
 import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat;
 import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
+import org.elasticsearch.simdvec.ESNextOSQVectorsScorer;
 import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -57,6 +59,9 @@ public class OSQScorerBenchmark {
     @Param({ "384", "782", "1024" })
     int dims;
 
+    @Param({ "1", "2", "4" })
+    int bits;
+
     int length;
 
     int numVectors = ES91OSQVectorsScorer.BULK_SIZE * 10;
@@ -68,8 +73,8 @@ public class OSQScorerBenchmark {
     float centroidDp;
 
     byte[] scratch;
-    ES91OSQVectorsScorer scorerMmap;
-    ES91OSQVectorsScorer scorerNfios;
+    ESNextOSQVectorsScorer scorerMmap;
+    ESNextOSQVectorsScorer scorerNfios;
 
     Directory dirMmap;
     IndexInput inMmap;
@@ -84,7 +89,12 @@ public class OSQScorerBenchmark {
     public void setup() throws IOException {
         Random random = new Random(123);
 
-        this.length = OptimizedScalarQuantizer.discretize(dims, 64) / 8;
+        this.length = switch (bits) {
+            case 1 -> ESNextDiskBBQVectorsFormat.QuantEncoding.ONE_BIT_4BIT_QUERY.getDocPackedLength(dims);
+            case 2 -> ESNextDiskBBQVectorsFormat.QuantEncoding.TWO_BIT_4BIT_QUERY.getDocPackedLength(dims);
+            case 4 -> ESNextDiskBBQVectorsFormat.QuantEncoding.FOUR_BIT_SYMMETRIC.getDocPackedLength(dims);
+            default -> throw new IllegalArgumentException("Unsupported bits: " + bits);
+        };
 
         binaryVectors = new byte[numVectors][length];
         for (byte[] binaryVector : binaryVectors) {
@@ -109,8 +119,14 @@ public void setup() throws IOException {
         outNfios.close();
         inMmap = dirMmap.openInput("vectors", IOContext.DEFAULT);
         inNiofs = dirNiofs.openInput("vectors", IOContext.DEFAULT);
-
-        binaryQueries = new byte[numVectors][4 * length];
+        int binaryQueryLength = switch (bits) {
+            case 1 -> ESNextDiskBBQVectorsFormat.QuantEncoding.ONE_BIT_4BIT_QUERY.getQueryPackedLength(dims);
+            case 2 -> ESNextDiskBBQVectorsFormat.QuantEncoding.TWO_BIT_4BIT_QUERY.getQueryPackedLength(dims);
+            case 4 -> ESNextDiskBBQVectorsFormat.QuantEncoding.FOUR_BIT_SYMMETRIC.getQueryPackedLength(dims);
+            default -> throw new IllegalArgumentException("Unsupported bits: " + bits);
+        };
+
+        binaryQueries = new byte[numVectors][binaryQueryLength];
         for (byte[] binaryVector : binaryVectors) {
             random.nextBytes(binaryVector);
         }
@@ -123,8 +139,26 @@ public void setup() throws IOException {
         centroidDp = random.nextFloat();
 
         scratch = new byte[length];
-        scorerMmap = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inMmap, dims);
-        scorerNfios = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inNiofs, dims);
+        final int docBits;
+        final int queryBits = switch (bits) {
+            case 1 -> {
+                docBits = 1;
+                yield 4;
+            }
+            case 2 -> {
+                docBits = 2;
+                yield 4;
+            }
+            case 4 -> {
+                docBits = 4;
+                yield 4;
+            }
+            default -> throw new IllegalArgumentException("Unsupported bits: " + bits);
+        };
+        scorerMmap = ESVectorizationProvider.getInstance()
+            .newESNextOSQVectorsScorer(inMmap, (byte) queryBits, (byte) docBits, dims, length);
+        scorerNfios = ESVectorizationProvider.getInstance()
+            .newESNextOSQVectorsScorer(inNiofs, (byte) queryBits, (byte) docBits, dims, length);
         scratchScores = new float[16];
         corrections = new float[3];
     }
@@ -156,7 +190,7 @@ public void scoreFromMemorySegmentOnlyVectorNiofsVect(Blackhole bh) throws IOExc
         scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
     }
 
-    private void scoreFromMemorySegmentOnlyVector(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
+    private void scoreFromMemorySegmentOnlyVector(Blackhole bh, IndexInput in, ESNextOSQVectorsScorer scorer) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i++) {
@@ -203,7 +237,7 @@ public void scoreFromMemorySegmentOnlyVectorBulkNiofsVect(Blackhole bh) throws I
         scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
     }
 
-    private void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
+    private void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh, IndexInput in, ESNextOSQVectorsScorer scorer) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i += 16) {
@@ -252,7 +286,7 @@ public void scoreFromMemorySegmentAllBulkNiofsVect(Blackhole bh) throws IOExcept
         scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
     }
 
-    private void scoreFromMemorySegmentAllBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
+    private void scoreFromMemorySegmentAllBulk(Blackhole bh, IndexInput in, ESNextOSQVectorsScorer scorer) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i += 16) {

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java

@@ -44,8 +44,9 @@ public void apply(Project project) {
         Provider<String> agplLicenseURL = licenseCommitProvider.map(
             licenseCommit -> ELASTIC_LICENSE_URL_PREFIX + licenseCommit + AGPL_ELASTIC_LICENSE_URL_POSTFIX
         );
-        // But stick the Elastic license url in project.ext so we can get it if we need to switch to it
+        // Stick the license urls in project.ext so we can get them if we need to switch to them
         project.getExtensions().getExtraProperties().set("elasticLicenseUrl", elasticLicenseURL);
+        project.getExtensions().getExtraProperties().set("agplLicenseUrl", agplLicenseURL);
 
         MapProperty<String, Provider<String>> licensesProperty = project.getObjects()
             .mapProperty(String.class, (Class<Provider<String>>) (Class<?>) Provider.class)

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=b84e04fa845fecba48551f425957641074fcc00a88a84d2aae5808743b35fc85
-distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-all.zip
+distributionSha256Sum=16f2b95838c1ddcf7242b1c39e7bbbb43c842f1f1a1a0dc4959b6d4d68abcac3
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.0-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactBasePlugin.java

@@ -11,6 +11,7 @@
 
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.provider.ProviderFactory;
 
 import javax.inject.Inject;
@@ -25,6 +26,7 @@ public InternalTestArtifactBasePlugin(ProviderFactory providerFactory) {
 
     @Override
     public void apply(Project project) {
+        project.getPlugins().apply(JavaPlugin.class);
         project.getPlugins().apply(ElasticsearchJavaBasePlugin.class);
         project.getExtensions().create("testArtifacts", InternalTestArtifactExtension.class, project, providerFactory);
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/StandaloneRestTestPlugin.java

@@ -53,15 +53,15 @@ public void apply(final Project project) {
 
         // only setup tests to build
         SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-        final SourceSet testSourceSet = sourceSets.create("test");
+        final SourceSet testSourceSet = sourceSets.maybeCreate("test");
 
         project.getTasks().withType(Test.class).configureEach(test -> {
             test.setTestClassesDirs(testSourceSet.getOutput().getClassesDirs());
             test.setClasspath(testSourceSet.getRuntimeClasspath());
         });
 
         // create a compileOnly configuration as others might expect it
-        project.getConfigurations().create("compileOnly");
+        project.getConfigurations().maybeCreate("compileOnly");
         RestTestUtil.setupJavaRestTestDependenciesDefaults(project, testSourceSet);
 
         EclipseModel eclipse = project.getExtensions().getByType(EclipseModel.class);

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-9.1.0
+9.2.0

build.gradle

@@ -77,7 +77,7 @@ tasks.named('zipAggregation').configure {
   destinationDirectory.set(layout.buildDirectory.dir("distributions"));
 
   dependsOn gradle.includedBuild('build-tools').task(':zipElasticPublication')
-  from(zipTree(gradle.includedBuild('build-tools').task(':zipElasticPublication').resolveTask().archiveFile.get()))
+  from(zipTree("build-tools/build/nmcp/zip/publicationElastic.zip"))
 }
 
 /**

distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile

@@ -25,7 +25,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:dea9b73186c5565a5bfa820a68f63ff497fdc9f06a0afc777159d819c5f01dc4 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:a02c67d96cd6ec1b50a055f1f5515e0987b643d001d436088cefc02e5b786bf9 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -104,7 +104,7 @@ WORKDIR /usr/share/elasticsearch/config
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:dea9b73186c5565a5bfa820a68f63ff497fdc9f06a0afc777159d819c5f01dc4
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:a02c67d96cd6ec1b50a055f1f5515e0987b643d001d436088cefc02e5b786bf9
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +