Skip to content

Commit 50e45ac

Browse files
junweid62Junwei Dai
junweid62
and
Junwei Dai
authored
Add API Consistency Tests with ML-Common and Set Up Daily GitHub Action Trigger (#937)
* Added the rest api spec comparison test Signed-off-by: Junwei Dai <[email protected]> * Added github Action to run api consistency test daily Signed-off-by: Junwei Dai <[email protected]> * fix formating issue Signed-off-by: Junwei Dai <[email protected]> * add change log Signed-off-by: Junwei Dai <[email protected]> * fix *import Signed-off-by: Junwei Dai <[email protected]> * addrssed all comment Signed-off-by: Junwei Dai <[email protected]> --------- Signed-off-by: Junwei Dai <[email protected]> Co-authored-by: Junwei Dai <[email protected]>
1 parent e5d08c5 commit 50e45ac

9 files changed

+129
-2
lines changed

‎.github/workflows/test-api-consistency.yml

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Daily API Consistency Test
2+
3+
on:
4+
schedule:
5+
- cron: '0 8 * * *' # Runs daily at 8 AM UTC
6+
workflow_dispatch:
7+
8+
jobs:
9+
API-consistency-test:
10+
runs-on: ubuntu-latest
11+
strategy:
12+
matrix:
13+
java: [21]
14+
15+
steps:
16+
- name: Checkout Flow Framework
17+
uses: actions/checkout@v3
18+
19+
- name: Setup Java ${{ matrix.java }}
20+
uses: actions/setup-java@v3
21+
with:
22+
distribution: 'temurin'
23+
java-version: ${{ matrix.java }}
24+
25+
- name: Run API Consistency Tests
26+
run: ./gradlew test --tests "org.opensearch.flowframework.workflow.*"

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
2121
### Features
2222
- Add ApiSpecFetcher for Fetching and Comparing API Specifications ([#651](https://github.com/opensearch-project/flow-framework/issues/651))
2323
- Add optional config field to tool step ([#899](https://github.com/opensearch-project/flow-framework/pull/899))
24+
- Add API Consistency Tests with ML-Common and Set Up Daily GitHub Action Trigger([#908](https://github.com/opensearch-project/flow-framework/issues/908))
2425

2526
### Enhancements
2627
- Incrementally remove resources from workflow state during deprovisioning ([#898](https://github.com/opensearch-project/flow-framework/pull/898))

‎src/main/java/org/opensearch/flowframework/util/ApiSpecFetcher.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.opensearch.flowframework.exception.ApiSpecParseException;
1515
import org.opensearch.rest.RestRequest;
1616

17-
import java.util.HashSet;
1817
import java.util.List;
1918

2019
import io.swagger.v3.oas.models.OpenAPI;
@@ -81,7 +80,7 @@ public static boolean compareRequiredFields(List<String> requiredEnumParams, Str
8180

8281
List<String> requiredApiParams = schema.getRequired();
8382
if (requiredApiParams != null && !requiredApiParams.isEmpty()) {
84-
return new HashSet<>(requiredEnumParams).equals(new HashSet<>(requiredApiParams));
83+
return requiredApiParams.stream().allMatch(requiredEnumParams::contains);
8584
}
8685
}
8786
return false;

‎src/test/java/org/opensearch/flowframework/workflow/CreateConnectorStepTests.java

+17
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,24 @@
1414
import org.opensearch.flowframework.common.CommonValue;
1515
import org.opensearch.flowframework.exception.FlowFrameworkException;
1616
import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
17+
import org.opensearch.flowframework.util.ApiSpecFetcher;
1718
import org.opensearch.ml.client.MachineLearningNodeClient;
1819
import org.opensearch.ml.common.connector.ConnectorAction;
1920
import org.opensearch.ml.common.transport.connector.MLCreateConnectorInput;
2021
import org.opensearch.ml.common.transport.connector.MLCreateConnectorResponse;
22+
import org.opensearch.rest.RestRequest;
2123
import org.opensearch.test.OpenSearchTestCase;
2224

2325
import java.io.IOException;
2426
import java.util.Collections;
27+
import java.util.List;
2528
import java.util.Map;
2629
import java.util.concurrent.ExecutionException;
2730

2831
import org.mockito.Mock;
2932
import org.mockito.MockitoAnnotations;
3033

34+
import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
3135
import static org.opensearch.flowframework.common.WorkflowResources.CONNECTOR_ID;
3236
import static org.mockito.ArgumentMatchers.any;
3337
import static org.mockito.ArgumentMatchers.anyString;
@@ -134,4 +138,17 @@ public void testCreateConnectorFailure() throws IOException {
134138
assertEquals("Failed to create connector", ex.getCause().getMessage());
135139
}
136140

141+
public void testApiSpecCreateConnectorInputParamComparison() throws Exception {
142+
List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.CREATE_CONNECTOR.inputs();
143+
144+
boolean isMatch = ApiSpecFetcher.compareRequiredFields(
145+
requiredEnumParams,
146+
ML_COMMONS_API_SPEC_YAML_URI,
147+
"/_plugins/_ml/connectors/_create",
148+
RestRequest.Method.POST
149+
);
150+
151+
assertTrue(isMatch);
152+
}
153+
137154
}

‎src/test/java/org/opensearch/flowframework/workflow/RegisterLocalCustomModelStepTests.java

+17
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@
2020
import org.opensearch.flowframework.exception.FlowFrameworkException;
2121
import org.opensearch.flowframework.exception.WorkflowStepException;
2222
import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
23+
import org.opensearch.flowframework.util.ApiSpecFetcher;
2324
import org.opensearch.ml.client.MachineLearningNodeClient;
2425
import org.opensearch.ml.common.MLTask;
2526
import org.opensearch.ml.common.MLTaskState;
2627
import org.opensearch.ml.common.transport.register.MLRegisterModelInput;
2728
import org.opensearch.ml.common.transport.register.MLRegisterModelResponse;
29+
import org.opensearch.rest.RestRequest;
2830
import org.opensearch.test.OpenSearchTestCase;
2931
import org.opensearch.threadpool.ScalingExecutorBuilder;
3032
import org.opensearch.threadpool.TestThreadPool;
@@ -33,6 +35,7 @@
3335

3436
import java.io.IOException;
3537
import java.util.Collections;
38+
import java.util.List;
3639
import java.util.Map;
3740
import java.util.concurrent.ExecutionException;
3841
import java.util.concurrent.TimeUnit;
@@ -43,6 +46,7 @@
4346

4447
import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD;
4548
import static org.opensearch.flowframework.common.CommonValue.FLOW_FRAMEWORK_THREAD_POOL_PREFIX;
49+
import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
4650
import static org.opensearch.flowframework.common.CommonValue.PROVISION_WORKFLOW_THREAD_POOL;
4751
import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS;
4852
import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_THREAD_POOL;
@@ -398,4 +402,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup
398402
assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage());
399403
assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus());
400404
}
405+
406+
public void testApiSpecRegisterLocalCustomModelInputParamComparison() throws Exception {
407+
List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_LOCAL_CUSTOM_MODEL.inputs();
408+
409+
boolean isMatch = ApiSpecFetcher.compareRequiredFields(
410+
requiredEnumParams,
411+
ML_COMMONS_API_SPEC_YAML_URI,
412+
"/_plugins/_ml/models/_register",
413+
RestRequest.Method.POST
414+
);
415+
416+
assertTrue(isMatch);
417+
}
401418
}

‎src/test/java/org/opensearch/flowframework/workflow/RegisterLocalPretrainedModelStepTests.java

+17
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@
2020
import org.opensearch.flowframework.exception.FlowFrameworkException;
2121
import org.opensearch.flowframework.exception.WorkflowStepException;
2222
import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
23+
import org.opensearch.flowframework.util.ApiSpecFetcher;
2324
import org.opensearch.ml.client.MachineLearningNodeClient;
2425
import org.opensearch.ml.common.MLTask;
2526
import org.opensearch.ml.common.MLTaskState;
2627
import org.opensearch.ml.common.transport.register.MLRegisterModelInput;
2728
import org.opensearch.ml.common.transport.register.MLRegisterModelResponse;
29+
import org.opensearch.rest.RestRequest;
2830
import org.opensearch.test.OpenSearchTestCase;
2931
import org.opensearch.threadpool.ScalingExecutorBuilder;
3032
import org.opensearch.threadpool.TestThreadPool;
@@ -33,6 +35,7 @@
3335

3436
import java.io.IOException;
3537
import java.util.Collections;
38+
import java.util.List;
3639
import java.util.Map;
3740
import java.util.concurrent.ExecutionException;
3841
import java.util.concurrent.TimeUnit;
@@ -42,6 +45,7 @@
4245

4346
import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD;
4447
import static org.opensearch.flowframework.common.CommonValue.FLOW_FRAMEWORK_THREAD_POOL_PREFIX;
48+
import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
4549
import static org.opensearch.flowframework.common.CommonValue.PROVISION_WORKFLOW_THREAD_POOL;
4650
import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS;
4751
import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_THREAD_POOL;
@@ -303,4 +307,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup
303307
assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage());
304308
assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus());
305309
}
310+
311+
public void testApiSpecRegisterLocalPretrainedModelInputParamComparison() throws Exception {
312+
List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_LOCAL_PRETRAINED_MODEL.inputs();
313+
314+
boolean isMatch = ApiSpecFetcher.compareRequiredFields(
315+
requiredEnumParams,
316+
ML_COMMONS_API_SPEC_YAML_URI,
317+
"/_plugins/_ml/models/_register",
318+
RestRequest.Method.POST
319+
);
320+
321+
assertTrue(isMatch);
322+
}
306323
}

‎src/test/java/org/opensearch/flowframework/workflow/RegisterLocalSparseEncodingModelStepTests.java

+17
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@
2020
import org.opensearch.flowframework.exception.FlowFrameworkException;
2121
import org.opensearch.flowframework.exception.WorkflowStepException;
2222
import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
23+
import org.opensearch.flowframework.util.ApiSpecFetcher;
2324
import org.opensearch.ml.client.MachineLearningNodeClient;
2425
import org.opensearch.ml.common.MLTask;
2526
import org.opensearch.ml.common.MLTaskState;
2627
import org.opensearch.ml.common.transport.register.MLRegisterModelInput;
2728
import org.opensearch.ml.common.transport.register.MLRegisterModelResponse;
29+
import org.opensearch.rest.RestRequest;
2830
import org.opensearch.test.OpenSearchTestCase;
2931
import org.opensearch.threadpool.ScalingExecutorBuilder;
3032
import org.opensearch.threadpool.TestThreadPool;
@@ -33,6 +35,7 @@
3335

3436
import java.io.IOException;
3537
import java.util.Collections;
38+
import java.util.List;
3639
import java.util.Map;
3740
import java.util.concurrent.ExecutionException;
3841
import java.util.concurrent.TimeUnit;
@@ -42,6 +45,7 @@
4245

4346
import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD;
4447
import static org.opensearch.flowframework.common.CommonValue.FLOW_FRAMEWORK_THREAD_POOL_PREFIX;
48+
import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
4549
import static org.opensearch.flowframework.common.CommonValue.PROVISION_WORKFLOW_THREAD_POOL;
4650
import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS;
4751
import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_THREAD_POOL;
@@ -310,4 +314,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup
310314
assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage());
311315
assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus());
312316
}
317+
318+
public void testApiSpecRegisterLocalSparseEncodingModelInputParamComparison() throws Exception {
319+
List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_LOCAL_SPARSE_ENCODING_MODEL.inputs();
320+
321+
boolean isMatch = ApiSpecFetcher.compareRequiredFields(
322+
requiredEnumParams,
323+
ML_COMMONS_API_SPEC_YAML_URI,
324+
"/_plugins/_ml/models/_register",
325+
RestRequest.Method.POST
326+
);
327+
328+
assertTrue(isMatch);
329+
}
313330
}

‎src/test/java/org/opensearch/flowframework/workflow/RegisterModelGroupStepTests.java

+16
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@
1414
import org.opensearch.flowframework.exception.FlowFrameworkException;
1515
import org.opensearch.flowframework.exception.WorkflowStepException;
1616
import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
17+
import org.opensearch.flowframework.util.ApiSpecFetcher;
1718
import org.opensearch.ml.client.MachineLearningNodeClient;
1819
import org.opensearch.ml.common.AccessMode;
1920
import org.opensearch.ml.common.MLTaskState;
2021
import org.opensearch.ml.common.transport.model_group.MLRegisterModelGroupInput;
2122
import org.opensearch.ml.common.transport.model_group.MLRegisterModelGroupResponse;
23+
import org.opensearch.rest.RestRequest;
2224
import org.opensearch.test.OpenSearchTestCase;
2325

2426
import java.io.IOException;
@@ -31,6 +33,7 @@
3133
import org.mockito.Mock;
3234
import org.mockito.MockitoAnnotations;
3335

36+
import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
3437
import static org.opensearch.flowframework.common.CommonValue.MODEL_GROUP_STATUS;
3538
import static org.opensearch.flowframework.common.WorkflowResources.MODEL_GROUP_ID;
3639
import static org.mockito.ArgumentMatchers.any;
@@ -204,4 +207,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup
204207
assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage());
205208
assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus());
206209
}
210+
211+
public void testApiSpecRegisterModelGroupInputParamComparison() throws Exception {
212+
List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_MODEL_GROUP.inputs();
213+
214+
boolean isMatch = ApiSpecFetcher.compareRequiredFields(
215+
requiredEnumParams,
216+
ML_COMMONS_API_SPEC_YAML_URI,
217+
"/_plugins/_ml/model_groups/_register",
218+
RestRequest.Method.POST
219+
);
220+
221+
assertTrue(isMatch);
222+
}
207223
}

‎src/test/java/org/opensearch/flowframework/workflow/RegisterRemoteModelStepTests.java

+17
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,18 @@
1717
import org.opensearch.flowframework.exception.FlowFrameworkException;
1818
import org.opensearch.flowframework.exception.WorkflowStepException;
1919
import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler;
20+
import org.opensearch.flowframework.util.ApiSpecFetcher;
2021
import org.opensearch.ml.client.MachineLearningNodeClient;
2122
import org.opensearch.ml.common.MLTaskState;
2223
import org.opensearch.ml.common.transport.register.MLRegisterModelInput;
2324
import org.opensearch.ml.common.transport.register.MLRegisterModelResponse;
25+
import org.opensearch.rest.RestRequest;
2426
import org.opensearch.test.OpenSearchTestCase;
2527
import org.opensearch.transport.RemoteTransportException;
2628

2729
import java.io.IOException;
2830
import java.util.Collections;
31+
import java.util.List;
2932
import java.util.Map;
3033
import java.util.concurrent.ExecutionException;
3134
import java.util.concurrent.atomic.AtomicInteger;
@@ -35,6 +38,7 @@
3538

3639
import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD;
3740
import static org.opensearch.flowframework.common.CommonValue.INTERFACE_FIELD;
41+
import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI;
3842
import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS;
3943
import static org.opensearch.flowframework.common.WorkflowResources.CONNECTOR_ID;
4044
import static org.opensearch.flowframework.common.WorkflowResources.MODEL_ID;
@@ -416,4 +420,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup
416420
assertEquals("Failed to parse value [yes] as only [true] or [false] are allowed.", w.getMessage());
417421
assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus());
418422
}
423+
424+
public void testApiSpecRegisterRemoteModelInputParamComparison() throws Exception {
425+
List<String> requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_REMOTE_MODEL.inputs();
426+
427+
boolean isMatch = ApiSpecFetcher.compareRequiredFields(
428+
requiredEnumParams,
429+
ML_COMMONS_API_SPEC_YAML_URI,
430+
"/_plugins/_ml/model_groups/_register",
431+
RestRequest.Method.POST
432+
);
433+
434+
assertTrue(isMatch);
435+
}
419436
}

0 commit comments

Comments
 (0)