Used DescriptiveStatistics and added validations for combination technique

owaiskazi19 · owaiskazi19 · commit 5ae7034614ba · 2025-03-13T14:44:15.000-07:00
Signed-off-by: Owais &lt;owaiskazi19@gmail.com&gt;
diff --git a/build.gradle b/build.gradle
@@ -260,6 +260,7 @@ dependencies {
     api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
     testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}"
     implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
+    implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
     // ml-common excluded reflection for runtime so we need to add it by ourselves.
     // https://github.com/opensearch-project/ml-commons/commit/464bfe34c66d7a729a00dd457f03587ea4e504d9
     // TODO: Remove following three lines of dependencies if ml-common include them in their jar
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ArithmeticMeanScoreCombinationTechnique.java
@@ -58,6 +58,11 @@ public float combine(final float[] scores) {
         return combinedScore / sumOfWeights;
     }
 
+    @Override
+    public String techniqueName() {
+        return TECHNIQUE_NAME;
+    }
+
     @Override
     public String describe() {
         return describeCombinationTechnique(TECHNIQUE_NAME, weights);
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/GeometricMeanScoreCombinationTechnique.java
@@ -58,6 +58,11 @@ public float combine(final float[] scores) {
         return sumOfWeights == 0 ? ZERO_SCORE : (float) Math.exp(weightedLnSum / sumOfWeights);
     }
 
+    @Override
+    public String techniqueName() {
+        return TECHNIQUE_NAME;
+    }
+
     @Override
     public String describe() {
         return describeCombinationTechnique(TECHNIQUE_NAME, weights);
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/HarmonicMeanScoreCombinationTechnique.java
@@ -55,6 +55,11 @@ public float combine(final float[] scores) {
         return sumOfHarmonics > 0 ? sumOfWeights / sumOfHarmonics : ZERO_SCORE;
     }
 
+    @Override
+    public String techniqueName() {
+        return TECHNIQUE_NAME;
+    }
+
     @Override
     public String describe() {
         return describeCombinationTechnique(TECHNIQUE_NAME, weights);
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/RRFScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/RRFScoreCombinationTechnique.java
@@ -37,6 +37,11 @@ public float combine(final float[] scores) {
         return sumScores;
     }
 
+    @Override
+    public String techniqueName() {
+        return TECHNIQUE_NAME;
+    }
+
     @Override
     public String describe() {
         return describeCombinationTechnique(TECHNIQUE_NAME, List.of());
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/combination/ScoreCombinationTechnique.java
@@ -12,4 +12,6 @@ public interface ScoreCombinationTechnique {
      * @return combined score
      */
     float combine(final float[] scores);
+
+    String techniqueName();
 }
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java b/src/main/java/org/opensearch/neuralsearch/processor/factory/NormalizationProcessorFactory.java
@@ -18,7 +18,6 @@
 import org.opensearch.neuralsearch.processor.normalization.MinMaxScoreNormalizationTechnique;
 import org.opensearch.neuralsearch.processor.normalization.ScoreNormalizationFactory;
 import org.opensearch.neuralsearch.processor.normalization.ScoreNormalizationTechnique;
-import org.opensearch.neuralsearch.processor.normalization.ZScoreNormalizationTechnique;
 import org.opensearch.search.pipeline.Processor;
 import org.opensearch.search.pipeline.SearchPhaseResultsProcessor;
 
@@ -51,9 +50,8 @@ public SearchPhaseResultsProcessor create(
     ) throws Exception {
         Map<String, Object> normalizationClause = readOptionalMap(NormalizationProcessor.TYPE, tag, config, NORMALIZATION_CLAUSE);
         ScoreNormalizationTechnique normalizationTechnique = ScoreNormalizationFactory.DEFAULT_METHOD;
-        String normalizationTechniqueName = MinMaxScoreNormalizationTechnique.TECHNIQUE_NAME;
         if (Objects.nonNull(normalizationClause)) {
-            normalizationTechniqueName = readStringProperty(
+            String normalizationTechniqueName = readStringProperty(
                 NormalizationProcessor.TYPE,
                 tag,
                 normalizationClause,
@@ -75,15 +73,13 @@ public SearchPhaseResultsProcessor create(
                 TECHNIQUE,
                 ArithmeticMeanScoreCombinationTechnique.TECHNIQUE_NAME
             );
-            // case when technique is z score and combination is not arithmetic mean
-            if (normalizationTechniqueName.equals(ZScoreNormalizationTechnique.TECHNIQUE_NAME)
-                && !combinationTechnique.equals(ArithmeticMeanScoreCombinationTechnique.TECHNIQUE_NAME)) {
-                throw new IllegalArgumentException("Z Score supports only arithmetic_mean combination technique");
-            }
             // check for optional combination params
             Map<String, Object> combinationParams = readOptionalMap(NormalizationProcessor.TYPE, tag, combinationClause, PARAMETERS);
             scoreCombinationTechnique = scoreCombinationFactory.createCombination(combinationTechnique, combinationParams);
         }
+
+        normalizationTechnique.validateCombinationTechnique(scoreCombinationTechnique);
+
         log.info(
             "Creating search phase results processor of type [{}] with normalization [{}] and combination [{}]",
             NormalizationProcessor.TYPE,
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/L2ScoreNormalizationTechnique.java
@@ -18,6 +18,10 @@
 import org.opensearch.neuralsearch.processor.NormalizeScoresDTO;
 
 import lombok.ToString;
+import org.opensearch.neuralsearch.processor.combination.ArithmeticMeanScoreCombinationTechnique;
+import org.opensearch.neuralsearch.processor.combination.GeometricMeanScoreCombinationTechnique;
+import org.opensearch.neuralsearch.processor.combination.HarmonicMeanScoreCombinationTechnique;
+import org.opensearch.neuralsearch.processor.combination.ScoreCombinationTechnique;
 import org.opensearch.neuralsearch.processor.explain.DocIdAtSearchShard;
 import org.opensearch.neuralsearch.processor.explain.ExplanationDetails;
 import org.opensearch.neuralsearch.processor.explain.ExplainableTechnique;
@@ -69,6 +73,21 @@ public void normalize(final NormalizeScoresDTO normalizeScoresDTO) {
         }
     }
 
+    @Override
+    public void validateCombinationTechnique(ScoreCombinationTechnique combinationTechnique) {
+        switch (combinationTechnique.techniqueName()) {
+            case ArithmeticMeanScoreCombinationTechnique.TECHNIQUE_NAME, GeometricMeanScoreCombinationTechnique.TECHNIQUE_NAME,
+                HarmonicMeanScoreCombinationTechnique.TECHNIQUE_NAME:
+                // These are the supported technique, so we do nothing
+                break;
+            default:
+                throw new IllegalArgumentException(
+                    "Z Score does not support the provided combination technique {}: Supported techniques are arithmetic_mean, geometric_mean and harmonic_mean"
+                        + combinationTechnique.techniqueName()
+                );
+        }
+    }
+
     @Override
     public String describe() {
         return String.format(Locale.ROOT, "%s", TECHNIQUE_NAME);
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/MinMaxScoreNormalizationTechnique.java
@@ -27,6 +27,10 @@
 
 import lombok.ToString;
 import org.opensearch.neuralsearch.processor.NormalizeScoresDTO;
+import org.opensearch.neuralsearch.processor.combination.ArithmeticMeanScoreCombinationTechnique;
+import org.opensearch.neuralsearch.processor.combination.GeometricMeanScoreCombinationTechnique;
+import org.opensearch.neuralsearch.processor.combination.HarmonicMeanScoreCombinationTechnique;
+import org.opensearch.neuralsearch.processor.combination.ScoreCombinationTechnique;
 import org.opensearch.neuralsearch.processor.explain.DocIdAtSearchShard;
 import org.opensearch.neuralsearch.processor.explain.ExplanationDetails;
 import org.opensearch.neuralsearch.processor.explain.ExplainableTechnique;
@@ -126,6 +130,21 @@ private MinMaxScores getMinMaxScoresResult(final List<CompoundTopDocs> queryTopD
         return new MinMaxScores(minScoresPerSubquery, maxScoresPerSubquery);
     }
 
+    @Override
+    public void validateCombinationTechnique(ScoreCombinationTechnique combinationTechnique) {
+        switch (combinationTechnique.techniqueName()) {
+            case ArithmeticMeanScoreCombinationTechnique.TECHNIQUE_NAME, GeometricMeanScoreCombinationTechnique.TECHNIQUE_NAME,
+                HarmonicMeanScoreCombinationTechnique.TECHNIQUE_NAME:
+                // These are the supported technique, so we do nothing
+                break;
+            default:
+                throw new IllegalArgumentException(
+                    "Z Score does not support the provided combination technique {}: Supported techniques are arithmetic_mean, geometric_mean and harmonic_mean"
+                        + combinationTechnique.techniqueName()
+                );
+        }
+    }
+
     @Override
     public String describe() {
         return lowerBoundsOptional.map(lb -> {
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/RRFNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/RRFNormalizationTechnique.java
@@ -24,6 +24,10 @@
 import lombok.ToString;
 import org.opensearch.neuralsearch.processor.NormalizeScoresDTO;
 import org.opensearch.neuralsearch.processor.SearchShard;
+import org.opensearch.neuralsearch.processor.combination.ArithmeticMeanScoreCombinationTechnique;
+import org.opensearch.neuralsearch.processor.combination.GeometricMeanScoreCombinationTechnique;
+import org.opensearch.neuralsearch.processor.combination.HarmonicMeanScoreCombinationTechnique;
+import org.opensearch.neuralsearch.processor.combination.ScoreCombinationTechnique;
 import org.opensearch.neuralsearch.processor.explain.DocIdAtSearchShard;
 import org.opensearch.neuralsearch.processor.explain.ExplainableTechnique;
 import org.opensearch.neuralsearch.processor.explain.ExplanationDetails;
@@ -76,6 +80,21 @@ public String describe() {
         return String.format(Locale.ROOT, "%s, rank_constant [%s]", TECHNIQUE_NAME, rankConstant);
     }
 
+    @Override
+    public void validateCombinationTechnique(ScoreCombinationTechnique combinationTechnique) {
+        switch (combinationTechnique.techniqueName()) {
+            case ArithmeticMeanScoreCombinationTechnique.TECHNIQUE_NAME, GeometricMeanScoreCombinationTechnique.TECHNIQUE_NAME,
+                HarmonicMeanScoreCombinationTechnique.TECHNIQUE_NAME:
+                // These are the supported technique, so we do nothing
+                break;
+            default:
+                throw new IllegalArgumentException(
+                    "Z Score does not support the provided combination technique {}: Supported techniques are arithmetic_mean, geometric_mean and harmonic_mean"
+                        + combinationTechnique.techniqueName()
+                );
+        }
+    }
+
     @Override
     public Map<DocIdAtSearchShard, ExplanationDetails> explain(List<CompoundTopDocs> queryTopDocs) {
         Map<DocIdAtSearchShard, List<Float>> normalizedScores = new HashMap<>();
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ScoreNormalizationTechnique.java
@@ -5,6 +5,7 @@
 package org.opensearch.neuralsearch.processor.normalization;
 
 import org.opensearch.neuralsearch.processor.NormalizeScoresDTO;
+import org.opensearch.neuralsearch.processor.combination.ScoreCombinationTechnique;
 
 /**
  * Abstracts normalization of scores in query search results.
@@ -20,4 +21,5 @@ public interface ScoreNormalizationTechnique {
      */
     void normalize(final NormalizeScoresDTO normalizeScoresDTO);
 
+    void validateCombinationTechnique(final ScoreCombinationTechnique combinationTechnique) throws IllegalArgumentException;
 }
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/normalization/ZScoreNormalizationTechnique.java b/src/main/java/org/opensearch/neuralsearch/processor/normalization/ZScoreNormalizationTechnique.java
diff --git a/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java b/src/test/java/org/opensearch/neuralsearch/processor/ScoreNormalizationIT.java
diff --git a/src/test/java/org/opensearch/neuralsearch/processor/normalization/ZScoreNormalizationTechniqueTests.java b/src/test/java/org/opensearch/neuralsearch/processor/normalization/ZScoreNormalizationTechniqueTests.java

Original file line number	Diff line number	Diff line change
`@@ -12,4 +12,6 @@ public interface ScoreCombinationTechnique {`
`12`	`12`	`* @return combined score`
`13`	`13`	`*/`
`14`	`14`	`float combine(final float[] scores);`
	`15`	`+`
	`16`	`+ String techniqueName();`
`15`	`17`	`}`