Add Import Capability for externally run Search Evaluation

.gitignore

@@ -70,3 +70,4 @@ testfixtures_shared/
 # These are generated from .ci/jobs.t
 .ci/jobs/
 src/test/scripts/esci_us_opensearch-2025-06-06.json
+src/test/data-external-evaluation/movies_experiment_tmp.json

CHANGELOG.md

@@ -8,6 +8,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Features
 
+### Removed
+
 ### Enhancements
 
 ### Bug Fixes

src/main/java/org/opensearch/searchrelevance/common/PluginConstants.java

@@ -82,6 +82,8 @@ private PluginConstants() {}
     public static final String QUERYSET_ID = "querySetId";
     public static final String SEARCH_CONFIGURATION_LIST = "searchConfigurationList";
     public static final String JUDGMENT_LIST = "judgmentList";
+    public static final String EVALUATION_RESULT_LIST = "evaluationResultList";
+    public static final String RESULTS = "results";
 
     public static final String JUDGMENT_RATINGS = "judgmentRatings";
     public static final String CONTEXT_FIELDS = "contextFields";

src/main/java/org/opensearch/searchrelevance/model/EvaluationResult.java

@@ -80,6 +80,19 @@ public EvaluationResult(
         this(id, timestamp, searchConfigurationId, searchText, judgmentIds, documentIds, metrics, null, null, null);
     }
 
+    public EvaluationResult(
+        String id,
+        String timestamp,
+        String experimentId,
+        String searchConfigurationId,
+        String searchText,
+        List<String> judgmentIds,
+        List<String> documentIds,
+        List<Map<String, Object>> metrics
+    ) {
+        this(id, timestamp, searchConfigurationId, searchText, judgmentIds, documentIds, metrics, experimentId, null, null);
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         XContentBuilder xContentBuilder = builder.startObject();

src/main/java/org/opensearch/searchrelevance/plugin/SearchRelevancePlugin.java

@@ -63,6 +63,7 @@
 import org.opensearch.searchrelevance.rest.RestGetJudgmentAction;
 import org.opensearch.searchrelevance.rest.RestGetQuerySetAction;
 import org.opensearch.searchrelevance.rest.RestGetSearchConfigurationAction;
+import org.opensearch.searchrelevance.rest.RestPostExperimentAction;
 import org.opensearch.searchrelevance.rest.RestPutExperimentAction;
 import org.opensearch.searchrelevance.rest.RestPutJudgmentAction;
 import org.opensearch.searchrelevance.rest.RestPutQuerySetAction;
@@ -75,6 +76,8 @@
 import org.opensearch.searchrelevance.transport.experiment.DeleteExperimentTransportAction;
 import org.opensearch.searchrelevance.transport.experiment.GetExperimentAction;
 import org.opensearch.searchrelevance.transport.experiment.GetExperimentTransportAction;
+import org.opensearch.searchrelevance.transport.experiment.PostExperimentAction;
+import org.opensearch.searchrelevance.transport.experiment.PostExperimentTransportAction;
 import org.opensearch.searchrelevance.transport.experiment.PutExperimentAction;
 import org.opensearch.searchrelevance.transport.experiment.PutExperimentTransportAction;
 import org.opensearch.searchrelevance.transport.judgment.DeleteJudgmentAction;
@@ -210,6 +213,7 @@ public List<RestHandler> getRestHandlers(
             new RestPutSearchConfigurationAction(settingsAccessor),
             new RestDeleteSearchConfigurationAction(settingsAccessor),
             new RestGetSearchConfigurationAction(settingsAccessor),
+            new RestPostExperimentAction(settingsAccessor),
             new RestPutExperimentAction(settingsAccessor),
             new RestGetExperimentAction(settingsAccessor),
             new RestDeleteExperimentAction(settingsAccessor),
@@ -230,6 +234,7 @@ public List<RestHandler> getRestHandlers(
             new ActionHandler<>(PutSearchConfigurationAction.INSTANCE, PutSearchConfigurationTransportAction.class),
             new ActionHandler<>(DeleteSearchConfigurationAction.INSTANCE, DeleteSearchConfigurationTransportAction.class),
             new ActionHandler<>(GetSearchConfigurationAction.INSTANCE, GetSearchConfigurationTransportAction.class),
+            new ActionHandler<>(PostExperimentAction.INSTANCE, PostExperimentTransportAction.class),
             new ActionHandler<>(PutExperimentAction.INSTANCE, PutExperimentTransportAction.class),
             new ActionHandler<>(DeleteExperimentAction.INSTANCE, DeleteExperimentTransportAction.class),
             new ActionHandler<>(GetExperimentAction.INSTANCE, GetExperimentTransportAction.class),

src/main/java/org/opensearch/searchrelevance/rest/RestPostExperimentAction.java

@@ -0,0 +1,147 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.searchrelevance.rest;
+
+import static java.util.Collections.singletonList;
+import static org.opensearch.rest.RestRequest.Method.POST;
+import static org.opensearch.searchrelevance.common.PluginConstants.EVALUATION_RESULT_LIST;
+import static org.opensearch.searchrelevance.common.PluginConstants.EXPERIMENTS_URI;
+import static org.opensearch.searchrelevance.common.PluginConstants.JUDGMENT_LIST;
+import static org.opensearch.searchrelevance.common.PluginConstants.QUERYSET_ID;
+import static org.opensearch.searchrelevance.common.PluginConstants.RESULTS;
+import static org.opensearch.searchrelevance.common.PluginConstants.SEARCH_CONFIGURATION_LIST;
+import static org.opensearch.searchrelevance.common.PluginConstants.SIZE;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.opensearch.action.index.IndexResponse;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.rest.RestStatus;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.rest.BaseRestHandler;
+import org.opensearch.rest.BytesRestResponse;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.searchrelevance.exception.SearchRelevanceException;
+import org.opensearch.searchrelevance.model.ExperimentType;
+import org.opensearch.searchrelevance.settings.SearchRelevanceSettingsAccessor;
+import org.opensearch.searchrelevance.transport.experiment.PostExperimentAction;
+import org.opensearch.searchrelevance.transport.experiment.PostExperimentRequest;
+import org.opensearch.searchrelevance.utils.ParserUtils;
+import org.opensearch.transport.client.node.NodeClient;
+
+import lombok.AllArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+@Log4j2
+/**
+ * Rest Action to facilitate requests to import a experiment.
+ */
+@AllArgsConstructor
+public class RestPostExperimentAction extends BaseRestHandler {
+    private static final String POST_EXPERIMENT_ACTION = "post_experiment_action";
+    private SearchRelevanceSettingsAccessor settingsAccessor;
+
+    @Override
+    public String getName() {
+        return POST_EXPERIMENT_ACTION;
+    }
+
+    @Override
+    public List<Route> routes() {
+        return singletonList(new Route(POST, EXPERIMENTS_URI));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        if (!settingsAccessor.isWorkbenchEnabled()) {
+            return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.FORBIDDEN, "Search Relevance Workbench is disabled"));
+        }
+
+        PostExperimentRequest createRequest = null;
+        XContentParser parser = request.contentParser();
+        Map<String, Object> source = parser.map();
+
+        String typeString = (String) source.get("type");
+        ExperimentType type;
+        try {
+            type = ExperimentType.valueOf(typeString);
+        } catch (IllegalArgumentException | NullPointerException e) {
+            throw new IllegalArgumentException("Invalid or missing experiment type", e);
+        }
+
+        if (type == ExperimentType.POINTWISE_EVALUATION) {
+            String querySetId = (String) source.get(QUERYSET_ID);
+            List<String> searchConfigurationList = ParserUtils.convertObjToList(source, SEARCH_CONFIGURATION_LIST);
+            if (searchConfigurationList.size() != 1) {
+                throw new IOException("Must have exactly one search configuration. Had " + searchConfigurationList.size() + " size.");
+            }
+            Integer sizeObj = (Integer) source.get(SIZE);
+            int size = sizeObj != null ? sizeObj.intValue() : 10; // Default size to 10 if not provided
+
+            List<String> judgmentList = ParserUtils.convertObjToList(source, JUDGMENT_LIST);
+            if (judgmentList.size() != 1) {
+                throw new IOException("Must have exactly one judgment list. Had " + judgmentList.size() + " size.");
+            }
+
+            List<Map<String, Object>> evaluationResultList = ParserUtils.convertObjToListOfMaps(source, EVALUATION_RESULT_LIST);
+
+            createRequest = new PostExperimentRequest(
+                type,
+                querySetId,
+                searchConfigurationList,
+                judgmentList,
+                size,
+                evaluationResultList,
+                null
+            );
+        } else if (type == ExperimentType.PAIRWISE_COMPARISON) {
+            String querySetId = (String) source.get(QUERYSET_ID);
+            List<String> searchConfigurationList = ParserUtils.convertObjToList(source, SEARCH_CONFIGURATION_LIST);
+            if (searchConfigurationList.size() != 2) {
+                throw new IOException("Must have exactly two search configurations. Had " + searchConfigurationList.size() + " size.");
+            }
+            Integer sizeObj = (Integer) source.get(SIZE);
+            int size = sizeObj != null ? sizeObj.intValue() : 10; // Default size to 10 if not provided
+
+            List<Map<String, Object>> results = ParserUtils.convertObjToListOfMaps(source, RESULTS);
+
+            createRequest = new PostExperimentRequest(type, querySetId, searchConfigurationList, List.of(), size, null, results);
+        } else {
+            throw new SearchRelevanceException("Importing experimentType" + type + " is not supported", RestStatus.BAD_REQUEST);
+        }
+
+        PostExperimentRequest finalCreateRequest = createRequest;
+        return channel -> client.execute(PostExperimentAction.INSTANCE, finalCreateRequest, new ActionListener<IndexResponse>() {
+            @Override
+            public void onResponse(IndexResponse response) {
+                try {
+                    XContentBuilder builder = channel.newBuilder();
+                    builder.startObject();
+                    builder.field("experiment_id", response.getId());
+                    builder.field("experiment_result", response.getResult());
+                    builder.endObject();
+                    channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
+                } catch (IOException e) {
+                    onFailure(e);
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                try {
+                    channel.sendResponse(new BytesRestResponse(channel, RestStatus.INTERNAL_SERVER_ERROR, e));
+                } catch (IOException ex) {
+                    log.error("Failed to send error response", ex);
+                }
+            }
+        });
+    }
+}

src/main/java/org/opensearch/searchrelevance/rest/RestPutExperimentAction.java

@@ -42,7 +42,7 @@
 
 @Log4j2
 /**
- * Rest Action to facilitate requests to create a experiment.
+ * Rest Action to facilitate requests to create an experiment.
  */
 @AllArgsConstructor
 public class RestPutExperimentAction extends BaseRestHandler {

src/main/java/org/opensearch/searchrelevance/transport/experiment/PostExperimentAction.java

@@ -0,0 +1,28 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.searchrelevance.transport.experiment;
+
+import static org.opensearch.searchrelevance.common.PluginConstants.TRANSPORT_ACTION_NAME_PREFIX;
+
+import org.opensearch.action.ActionType;
+import org.opensearch.action.index.IndexResponse;
+
+/**
+ * External Action for public facing RestPostExperimentAction
+ */
+public class PostExperimentAction extends ActionType<IndexResponse> {
+    /** The name of this action */
+    public static final String NAME = TRANSPORT_ACTION_NAME_PREFIX + "experiment/import";
+
+    /** An instance of this action */
+    public static final PostExperimentAction INSTANCE = new PostExperimentAction();
+
+    private PostExperimentAction() {
+        super(NAME, IndexResponse::new);
+    }
+}