HSEARCH-5064 Integrate Elasticsearch's knn query introduced in 8.12

Author: marko-bekhta

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/dialect/impl/ElasticsearchDialectFactory.java

@@ -12,6 +12,7 @@
 import org.hibernate.search.backend.elasticsearch.ElasticsearchDistributionName;
 import org.hibernate.search.backend.elasticsearch.ElasticsearchVersion;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch7ModelDialect;
+import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch812ModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch8ModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.ElasticsearchModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.OpenSearch1ModelDialect;
@@ -97,7 +98,12 @@ else if ( major == 7 ) {
 			return new Elasticsearch7ModelDialect();
 		}
 		else {
-			return new Elasticsearch8ModelDialect();
+			// if there's no minor -- who knows which version it is, better stay safe
+			// and assume that only 8 features are available, and nothing from 8.12+
+			if ( major == 8 && ( minorOptional.isEmpty() || minorOptional.getAsInt() < 12 ) ) {
+				return new Elasticsearch8ModelDialect();
+			}
+			return new Elasticsearch812ModelDialect();
 		}
 	}
 

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/dialect/model/impl/Elasticsearch812ModelDialect.java

@@ -0,0 +1,30 @@
+/*
+ * Hibernate Search, full-text search for your domain model
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
+ */
+package org.hibernate.search.backend.elasticsearch.dialect.model.impl;
+
+import org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl.Elasticsearch812IndexFieldTypeFactoryProvider;
+import org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl.ElasticsearchIndexFieldTypeFactoryProvider;
+import org.hibernate.search.backend.elasticsearch.validation.impl.Elasticsearch8PropertyMappingValidatorProvider;
+import org.hibernate.search.backend.elasticsearch.validation.impl.ElasticsearchPropertyMappingValidatorProvider;
+
+import com.google.gson.Gson;
+
+/**
+ * The model dialect for Elasticsearch 8.12+.
+ */
+public class Elasticsearch812ModelDialect implements ElasticsearchModelDialect {
+
+	@Override
+	public ElasticsearchIndexFieldTypeFactoryProvider createIndexTypeFieldFactoryProvider(Gson userFacingGson) {
+		return new Elasticsearch812IndexFieldTypeFactoryProvider( userFacingGson );
+	}
+
+	@Override
+	public ElasticsearchPropertyMappingValidatorProvider createElasticsearchPropertyMappingValidatorProvider() {
+		return new Elasticsearch8PropertyMappingValidatorProvider();
+	}
+}

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/dialect/model/impl/Elasticsearch8ModelDialect.java

@@ -14,7 +14,7 @@
 import com.google.gson.Gson;
 
 /**
- * The model dialect for Elasticsearch 8.x.
+ * The model dialect for Elasticsearch [8.0-8.11].
  */
 public class Elasticsearch8ModelDialect implements ElasticsearchModelDialect {
 

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/search/predicate/impl/ElasticsearchKnnPredicate.java

@@ -58,6 +58,19 @@ public KnnPredicateBuilder create(ElasticsearchSearchIndexScope<?> scope,
 		}
 	}
 
+	public static class Elasticsearch812Factory<F>
+			extends AbstractElasticsearchCodecAwareSearchQueryElementFactory<KnnPredicateBuilder, F> {
+		public Elasticsearch812Factory(ElasticsearchFieldCodec<F> codec) {
+			super( codec );
+		}
+
+		@Override
+		public KnnPredicateBuilder create(ElasticsearchSearchIndexScope<?> scope,
+				ElasticsearchSearchIndexValueFieldContext<F> field) {
+			return new Elasticsearch812Impl.Builder<>( codec, scope, field );
+		}
+	}
+
 	public static class OpenSearchFactory<F>
 			extends AbstractElasticsearchCodecAwareSearchQueryElementFactory<KnnPredicateBuilder, F> {
 		public OpenSearchFactory(ElasticsearchFieldCodec<F> codec) {
@@ -215,6 +228,58 @@ public SearchPredicate build() {
 		}
 	}
 
+	private static class Elasticsearch812Impl extends ElasticsearchKnnPredicate {
+
+		private static final JsonObjectAccessor KNN_ACCESSOR = JsonAccessor.root().property( "knn" ).asObject();
+		private static final JsonAccessor<String> FIELD_ACCESSOR = JsonAccessor.root().property( "field" ).asString();
+		private static final JsonArrayAccessor VECTOR_ACCESSOR = JsonAccessor.root().property( "query_vector" ).asArray();
+		private static final JsonObjectAccessor FILTER_ACCESSOR = JsonAccessor.root().property( "filter" ).asObject();
+		private static final JsonAccessor<Integer> NUM_CANDIDATES_ACCESSOR =
+				JsonAccessor.root().property( "num_candidates" ).asInteger();
+		private static final JsonAccessor<Float> SIMILARITY_ACCESSOR = JsonAccessor.root().property( "similarity" ).asFloat();
+
+
+		private Elasticsearch812Impl(Builder<?> builder) {
+			super( builder );
+		}
+
+		@Override
+		protected JsonObject doToJsonQuery(PredicateRequestContext context, JsonObject outerObject, JsonObject innerObject) {
+			KNN_ACCESSOR.set( outerObject, innerObject );
+
+			FIELD_ACCESSOR.set( innerObject, absoluteFieldPath );
+			NUM_CANDIDATES_ACCESSOR.set( innerObject, k );
+			VECTOR_ACCESSOR.set( innerObject, vector );
+
+			if ( filter != null ) {
+				FILTER_ACCESSOR.set( innerObject, filter.toJsonQuery( context ) );
+			}
+			if ( similarity != null ) {
+				SIMILARITY_ACCESSOR.set( innerObject, similarity );
+			}
+
+			return outerObject;
+		}
+
+		private static class Builder<F> extends AbstractKnnBuilder<F> {
+
+			private Builder(ElasticsearchFieldCodec<F> codec, ElasticsearchSearchIndexScope<?> scope,
+					ElasticsearchSearchIndexValueFieldContext<F> field) {
+				super( codec, scope, field );
+			}
+
+			@Override
+			public void requiredMinimumSimilarity(float similarity) {
+				this.similarity = similarity;
+			}
+
+			@Override
+			public SearchPredicate build() {
+				return new Elasticsearch812Impl( this );
+			}
+		}
+	}
+
 	private static class OpenSearchImpl extends ElasticsearchKnnPredicate {
 
 		private static final JsonObjectAccessor KNN_ACCESSOR = JsonAccessor.root().property( "knn" ).asObject();

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/dsl/provider/impl/Elasticsearch812IndexFieldTypeFactoryProvider.java

@@ -0,0 +1,30 @@
+/*
+ * Hibernate Search, full-text search for your domain model
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
+ */
+package org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl;
+
+import org.hibernate.search.backend.elasticsearch.types.mapping.impl.Elasticsearch812VectorFieldTypeMappingContributor;
+import org.hibernate.search.backend.elasticsearch.types.mapping.impl.ElasticsearchVectorFieldTypeMappingContributor;
+
+import com.google.gson.Gson;
+
+/**
+ * The index field type factory provider for ES8.12+.
+ */
+public class Elasticsearch812IndexFieldTypeFactoryProvider extends AbstractIndexFieldTypeFactoryProvider {
+
+	private final Elasticsearch812VectorFieldTypeMappingContributor vectorFieldTypeMappingContributor =
+			new Elasticsearch812VectorFieldTypeMappingContributor();
+
+	public Elasticsearch812IndexFieldTypeFactoryProvider(Gson userFacingGson) {
+		super( userFacingGson );
+	}
+
+	@Override
+	protected ElasticsearchVectorFieldTypeMappingContributor vectorFieldTypeMappingContributor() {
+		return vectorFieldTypeMappingContributor;
+	}
+}

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/dsl/provider/impl/Elasticsearch8IndexFieldTypeFactoryProvider.java

@@ -12,7 +12,7 @@
 import com.google.gson.Gson;
 
 /**
- * The index field type factory provider for ES8.x.
+ * The index field type factory provider for ES [8.0-8.11].
  */
 public class Elasticsearch8IndexFieldTypeFactoryProvider extends AbstractIndexFieldTypeFactoryProvider {
 

backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/mapping/impl/Elasticsearch812VectorFieldTypeMappingContributor.java

@@ -0,0 +1,19 @@
+/*
+ * Hibernate Search, full-text search for your domain model
+ *
+ * License: GNU Lesser General Public License (LGPL), version 2.1 or later
+ * See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
+ */
+package org.hibernate.search.backend.elasticsearch.types.mapping.impl;
+
+import org.hibernate.search.backend.elasticsearch.search.predicate.impl.ElasticsearchKnnPredicate;
+import org.hibernate.search.backend.elasticsearch.types.impl.ElasticsearchIndexValueFieldType;
+import org.hibernate.search.engine.search.predicate.spi.PredicateTypeKeys;
+
+public class Elasticsearch812VectorFieldTypeMappingContributor extends Elasticsearch8VectorFieldTypeMappingContributor {
+	@Override
+	public <F> void contribute(ElasticsearchIndexValueFieldType.Builder<F> builder, Context context) {
+		builder.queryElementFactory( PredicateTypeKeys.KNN,
+				new ElasticsearchKnnPredicate.Elasticsearch812Factory<>( builder.codec() ) );
+	}
+}

backend/elasticsearch/src/test/java/org/hibernate/search/backend/elasticsearch/dialect/impl/ElasticsearchDialectFactoryTest.java

@@ -15,6 +15,7 @@
 import org.hibernate.search.backend.elasticsearch.ElasticsearchDistributionName;
 import org.hibernate.search.backend.elasticsearch.ElasticsearchVersion;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch7ModelDialect;
+import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch812ModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.Elasticsearch8ModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.ElasticsearchModelDialect;
 import org.hibernate.search.backend.elasticsearch.dialect.model.impl.OpenSearch1ModelDialect;
@@ -241,23 +242,23 @@ public static List<? extends Arguments> params() {
 				),
 				success(
 						ElasticsearchDistributionName.ELASTIC, "8.12", "8.12.0",
-						Elasticsearch8ModelDialect.class, Elasticsearch81ProtocolDialect.class
+						Elasticsearch812ModelDialect.class, Elasticsearch81ProtocolDialect.class
 				),
 				success(
 						ElasticsearchDistributionName.ELASTIC, "8.12.0", "8.12.0",
-						Elasticsearch8ModelDialect.class, Elasticsearch81ProtocolDialect.class
+						Elasticsearch812ModelDialect.class, Elasticsearch81ProtocolDialect.class
 				),
 				successWithWarning(
 						ElasticsearchDistributionName.ELASTIC, "8.13", "8.13.0",
-						Elasticsearch8ModelDialect.class, Elasticsearch81ProtocolDialect.class
+						Elasticsearch812ModelDialect.class, Elasticsearch81ProtocolDialect.class
 				),
 				successWithWarning(
 						ElasticsearchDistributionName.ELASTIC, "8.13.0", "8.13.0",
-						Elasticsearch8ModelDialect.class, Elasticsearch81ProtocolDialect.class
+						Elasticsearch812ModelDialect.class, Elasticsearch81ProtocolDialect.class
 				),
 				successWithWarning(
 						ElasticsearchDistributionName.ELASTIC, "9.0.0", "9.0.0",
-						Elasticsearch8ModelDialect.class, Elasticsearch81ProtocolDialect.class
+						Elasticsearch812ModelDialect.class, Elasticsearch81ProtocolDialect.class
 				),
 				success(
 						ElasticsearchDistributionName.OPENSEARCH, "1", "1.3.1",

documentation/src/main/asciidoc/public/reference/_search-dsl-predicate.adoc

@@ -1578,7 +1578,7 @@ While when an <<backend-elasticsearch-compatibility-opensearch,OpenSearch>> dist
 when the index is configured to have more than one shard.
 See this section of the OpenSearch link:{openSearchDocUrl}/search-plugins/knn/approximate-knn/#get-started-with-approximate-k-nn[documentation] for more details.
 
-With the Elasticsearch backend and when using <<backend-elasticsearch-compatibility-elasticsearch,Elasticsearch>>,
+With the Elasticsearch backend and when using <<backend-elasticsearch-compatibility-elasticsearch,Elasticsearch>> version < 8.12,
 a `knn` predicate can only be added as a top-level predicate,
 i.e. a predicate directly passed to a where clause of a search query,
 or as part of a top-level disjunction,
@@ -1588,7 +1588,9 @@ A `knn` predicate can be combined with other predicate types by adding them thro
 Any other usages of a `knn` predicate, with this backend, would lead to an exception being thrown.
 See this section of the Elasticsearch link:{elasticsearchDocUrl}/knn-search.html#_combine_approximate_knn_with_other_features[documentation]
 in particular to learn more about how Elasticsearch combines regular queries with knn.
-<<backend-elasticsearch-compatibility-opensearch,OpenSearch>> does not have these limitation.
+For Elasticsearch 8.12+ a knn predicate is converted to a link:{elasticsearchDocUrl}/query-dsl-knn-query.html#query-dsl-knn-query[knn-query]
+removing the limitations described for earlier versions.
+<<backend-elasticsearch-compatibility-opensearch,OpenSearch>> also does not have these limitation.
 
 .Multiple `knn` predicates added via `should` clauses
 ====

documentation/src/test/java/org/hibernate/search/documentation/search/predicate/PredicateDslIT.java

@@ -1185,7 +1185,7 @@ void knn() {
 
 		if ( !BackendConfiguration.isElasticsearch()
 				|| ElasticsearchTestDialect.isActualVersion(
-						es -> false,
+						es -> !es.isLessThan( "8.12.0" ),
 						os -> !os.isLessThan( "2.0" ),
 						aoss -> true
 				) ) {

integrationtest/backend/elasticsearch/src/test/java/org/hibernate/search/integrationtest/backend/elasticsearch/testsupport/util/ElasticsearchTckBackendFeatures.java

@@ -250,16 +250,16 @@ public boolean supportsExplicitRefresh() {
 	@Override
 	public boolean supportsVectorSearch() {
 		return isActualVersion(
-				es -> !es.isLessThan( "8.0" ),
-				os -> !os.isLessThan( "2.0" ),
+				es -> !es.isLessThan( "8.0.0" ),
+				os -> !os.isLessThan( "2.0.0" ),
 				aoss -> true
 		);
 	}
 
 	@Override
 	public boolean supportsVectorSearchInsideOtherPredicates() {
 		return isActualVersion(
-				es -> false,
+				es -> !es.isLessThan( "8.12.0" ),
 				os -> true,
 				aoss -> true
 		);
@@ -301,7 +301,7 @@ public boolean supportsSimilarity(VectorSimilarity vectorSimilarity) {
 	@Override
 	public boolean supportsVectorSearchKPerShard() {
 		return isActualVersion(
-				es -> false,
+				es -> !es.isLessThan( "8.12.0" ),
 				os -> true,
 				aoss -> true
 		);