Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixing unit tests related to mlclient getTask and adding integration test for workflow provisioning under multitenancy #1045

Merged
merged 3 commits into from
Feb 4, 2025
Merged

Fixing unit tests related to mlclient getTask and adding integration test for workflow provisioning under multitenancy #1045

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
eb6cd64
Fixing unit tests related to mlclient getTask
Feb 4, 2025
6b1d2b1
Adding integration tests for workflow provisioning under multitenancy
Feb 4, 2025
6b79b68
Addressing comments
Feb 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,7 @@ integTest {
includeTestsMatching "org.opensearch.flowframework.*TenantAwareIT"
}
systemProperty "plugins.flow_framework.multi_tenancy_enabled", "true"
systemProperty "plugins.ml_commons.multi_tenancy_enabled", "true"
}

// Only rest case can run with remote cluster
Expand Down Expand Up @@ -316,6 +317,7 @@ integTest {
writer ->
writer.write("\n# Set multitenancy\n")
writer.write("plugins.flow_framework.multi_tenancy_enabled: true\n")
writer.write("plugins.ml_commons.multi_tenancy_enabled: true\n")
}
// TODO this properly uses the remote client factory but needs a remote cluster set up
// TODO get the endpoint from a system property
Expand Down
1 change: 1 addition & 0 deletions src/test/java/org/opensearch/flowframework/FlowFrameworkTenantAwareRestTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public abstract class FlowFrameworkTenantAwareRestTestCase extends FlowFramework
// REST Response error reasons
protected static final String MISSING_TENANT_REASON = "Tenant ID header is missing or has no value";
protected static final String NO_PERMISSION_REASON = "No permission to access this resource";
protected static final String NO_RESOURCE_ACCESS_PERMISSION_REASON = "You don't have permission to access this resource";

protected String tenantId = randomAlphaOfLength(5);
protected String otherTenantId = randomAlphaOfLength(6);
Expand Down
4 changes: 3 additions & 1 deletion src/test/java/org/opensearch/flowframework/rest/FlowFrameworkRestApiIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,13 +247,15 @@ public void testCreateAndProvisionRemoteModelWorkflow() throws Exception {
List<ResourceCreated> resourcesCreated = getResourcesCreated(client(), workflowId, 30);

// This template should create 3 resources, connector_id, registered model_id and deployed model_id
assertEquals(3, resourcesCreated.size());
assertEquals(4, resourcesCreated.size());
assertEquals("create_connector", resourcesCreated.get(0).workflowStepName());
assertNotNull(resourcesCreated.get(0).resourceId());
assertEquals("register_remote_model", resourcesCreated.get(1).workflowStepName());
assertNotNull(resourcesCreated.get(1).resourceId());
assertEquals("deploy_model", resourcesCreated.get(2).workflowStepName());
assertNotNull(resourcesCreated.get(2).resourceId());
assertEquals("register_agent", resourcesCreated.get(3).workflowStepName());
assertNotNull(resourcesCreated.get(3).resourceId());

// Delete the workflow without deleting the resources
Response deleteResponse = deleteWorkflow(client(), workflowId, "?clear_status=true");
Expand Down
200 changes: 200 additions & 0 deletions src/test/java/org/opensearch/flowframework/rest/RestWorkflowStateTenantAwareIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.opensearch.rest.RestRequest;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

Expand All @@ -32,6 +33,12 @@ public class RestWorkflowStateTenantAwareIT extends FlowFrameworkTenantAwareRest
private static final String STATUS_ALL = "/_status?all=true";
private static final String CLEAR_STATUS = "?clear_status=true";

// REST paths; some subclasses need multiple of these
private static final String AGENTS_PATH = "/_plugins/_ml/agents/";
private static final String CONNECTORS_PATH = "/_plugins/_ml/connectors/";
private static final String MODELS_PATH = "/_plugins/_ml/models/";
private static final String MODEL_GROUPS_PATH = "/_plugins/_ml/model_groups/";

public void testWorkflowStateCRUD() throws Exception {
boolean multiTenancyEnabled = isMultiTenancyEnabled();

Expand Down Expand Up @@ -158,6 +165,166 @@ public void testWorkflowStateCRUD() throws Exception {
assertEquals("COMPLETED", stateMap.get("state"));
}, 20, TimeUnit.SECONDS);

// Verify created resources and ml client calls for connector, model & agent
response = makeRequest(tenantRequest, GET, WORKFLOW_PATH + workflowId + STATUS_ALL);
assertOK(response);
Map<String, Object> statemap = responseToMap(response);
List<Map<String, Object>> resourcesCreated = (List<Map<String, Object>>) statemap.get("resources_created");
String connectorId = resourcesCreated.get(0).get("resource_id").toString();
assertNotNull(connectorId);

String modelId = resourcesCreated.get(1).get("resource_id").toString();
assertNotNull(modelId);

String modelGroupId = resourcesCreated.get(2).get("resource_id").toString();
assertNotNull(modelGroupId);

String agentId = resourcesCreated.get(3).get("resource_id").toString();
assertNotNull(agentId);

// verify ml client call for Connector with valid tenant
assertBusy(() -> {
Response restResponse = makeRequest(tenantRequest, GET, CONNECTORS_PATH + connectorId);
assertOK(restResponse);
Map<String, Object> mlCommonsResponseMap = responseToMap(restResponse);
if (multiTenancyEnabled) {
assertEquals(tenantId, mlCommonsResponseMap.get(TENANT_ID_FIELD));
} else {
assertNull(mlCommonsResponseMap.get(TENANT_ID_FIELD));
}
}, 20, TimeUnit.SECONDS);

// Now try again with an other ID
if (multiTenancyEnabled) {
ResponseException ex = assertThrows(
ResponseException.class,
() -> makeRequest(otherTenantRequest, GET, CONNECTORS_PATH + connectorId)
);
response = ex.getResponse();
map = responseToMap(response);
if (DDB) {
assertNotFound(response);
assertEquals("Failed to find connector with the provided connector id: " + connectorId, getErrorReasonFromResponseMap(map));
} else {
assertForbidden(response);
assertEquals(NO_RESOURCE_ACCESS_PERMISSION_REASON, getErrorReasonFromResponseMap(map));
}
} else {
response = makeRequest(otherTenantRequest, GET, CONNECTORS_PATH + connectorId);
assertOK(response);
map = responseToMap(response);
assertEquals("OpenAI Chat Connector", map.get("name"));
}

// Now try again with a null ID
if (multiTenancyEnabled) {
ResponseException ex = assertThrows(
ResponseException.class,
() -> makeRequest(nullTenantRequest, GET, CONNECTORS_PATH + connectorId)
);
response = ex.getResponse();
map = responseToMap(response);
assertForbidden(response);
assertEquals(MISSING_TENANT_REASON, getErrorReasonFromResponseMap(map));
} else {
response = makeRequest(nullTenantRequest, GET, CONNECTORS_PATH + connectorId);
assertOK(response);
map = responseToMap(response);
assertEquals("OpenAI Chat Connector", map.get("name"));
}

// verify ml client call for Model with valid tenant
assertBusy(() -> {
Response restResponse = makeRequest(tenantRequest, GET, MODELS_PATH + modelId);
assertOK(restResponse);
Map<String, Object> mlModelsResponseMap = responseToMap(restResponse);
assertEquals("test model", mlModelsResponseMap.get("description"));
if (multiTenancyEnabled) {
assertEquals(tenantId, mlModelsResponseMap.get(TENANT_ID_FIELD));
} else {
assertNull(mlModelsResponseMap.get(TENANT_ID_FIELD));
}
}, 20, TimeUnit.SECONDS);

// Now try again with an other ID
if (multiTenancyEnabled) {
ResponseException ex = assertThrows(ResponseException.class, () -> makeRequest(otherTenantRequest, GET, MODELS_PATH + modelId));
response = ex.getResponse();
map = responseToMap(response);
if (DDB) {
assertNotFound(response);
assertEquals("Failed to find model with the provided model id: " + modelId, getErrorReasonFromResponseMap(map));
} else {
assertForbidden(response);
assertEquals(NO_RESOURCE_ACCESS_PERMISSION_REASON, getErrorReasonFromResponseMap(map));
}
} else {
response = makeRequest(otherTenantRequest, GET, MODELS_PATH + modelId);
assertOK(response);
map = responseToMap(response);
assertEquals("test model", map.get("description"));
}

// Now try again with a null ID
if (multiTenancyEnabled) {
ResponseException ex = assertThrows(ResponseException.class, () -> makeRequest(nullTenantRequest, GET, MODELS_PATH + modelId));
response = ex.getResponse();
assertForbidden(response);
map = responseToMap(response);
assertEquals(MISSING_TENANT_REASON, getErrorReasonFromResponseMap(map));
} else {
response = makeRequest(nullTenantRequest, GET, MODELS_PATH + modelId);
assertOK(response);
map = responseToMap(response);
assertEquals("test model", map.get("description"));
}

// verify ml client call for Agent with valid tenant
assertBusy(() -> {
Response restResponse = makeRequest(tenantRequest, GET, AGENTS_PATH + agentId);
assertOK(restResponse);
Map<String, Object> mlModelsResponseMap = responseToMap(restResponse);
assertEquals("Test Agent", mlModelsResponseMap.get("name"));
if (multiTenancyEnabled) {
assertEquals(tenantId, mlModelsResponseMap.get(TENANT_ID_FIELD));
} else {
assertNull(mlModelsResponseMap.get(TENANT_ID_FIELD));
}
}, 20, TimeUnit.SECONDS);

// Now try again with an other ID
if (multiTenancyEnabled) {
ResponseException ex = assertThrows(ResponseException.class, () -> makeRequest(otherTenantRequest, GET, AGENTS_PATH + agentId));
response = ex.getResponse();
map = responseToMap(response);
if (DDB) {
assertNotFound(response);
assertEquals("Failed to find agent with the provided agent id: " + agentId, getErrorReasonFromResponseMap(map));
} else {
assertForbidden(response);
assertEquals(NO_RESOURCE_ACCESS_PERMISSION_REASON, getErrorReasonFromResponseMap(map));
}
} else {
response = makeRequest(otherTenantRequest, GET, AGENTS_PATH + agentId);
assertOK(response);
map = responseToMap(response);
assertEquals("Test Agent", map.get("name"));
}

// Now try again with a null ID
if (multiTenancyEnabled) {
ResponseException ex = assertThrows(ResponseException.class, () -> makeRequest(nullTenantRequest, GET, AGENTS_PATH + agentId));
response = ex.getResponse();
assertForbidden(response);
map = responseToMap(response);
assertEquals(MISSING_TENANT_REASON, getErrorReasonFromResponseMap(map));
} else {
response = makeRequest(nullTenantRequest, GET, AGENTS_PATH + agentId);
assertOK(response);
map = responseToMap(response);
assertEquals("Test Agent", map.get("name"));
}

/*
* Search
*/
Expand Down Expand Up @@ -311,6 +478,19 @@ public void testWorkflowStateCRUD() throws Exception {
assertOK(response);
map = responseToMap(response);
assertEquals("COMPLETED", map.get("state"));
resourcesCreated = (List<Map<String, Object>>) map.get("resources_created");

String otherConnectorId = resourcesCreated.get(0).get("resource_id").toString();
assertNotNull(otherConnectorId);

String otherModelId = resourcesCreated.get(1).get("resource_id").toString();
assertNotNull(otherModelId);

String otherModelGroupId = resourcesCreated.get(2).get("resource_id").toString();
assertNotNull(otherModelGroupId);

String otherAgentId = resourcesCreated.get(3).get("resource_id").toString();
assertNotNull(otherAgentId);

// Now finally deprovision the right way
response = makeRequest(otherTenantRequest, POST, WORKFLOW_PATH + otherWorkflowId + DEPROVISION);
Expand All @@ -328,6 +508,26 @@ public void testWorkflowStateCRUD() throws Exception {
assertEquals("NOT_STARTED", stateMap.get("state"));
}, 20, TimeUnit.SECONDS);

// verify if resources are deleted after de-provisioning
ex = assertThrows(ResponseException.class, () -> makeRequest(otherTenantRequest, GET, CONNECTORS_PATH + otherConnectorId));
response = ex.getResponse();
map = responseToMap(response);
assertNotFound(response);
assertEquals("Failed to find connector with the provided connector id: " + otherConnectorId, getErrorReasonFromResponseMap(map));

ex = assertThrows(ResponseException.class, () -> makeRequest(otherTenantRequest, GET, MODELS_PATH + otherModelId));
response = ex.getResponse();
assertNotFound(response);
map = responseToMap(response);
assertEquals("Failed to find model with the provided model id: " + otherModelId, getErrorReasonFromResponseMap(map));

// Verify the deletion
ex = assertThrows(ResponseException.class, () -> makeRequest(otherTenantRequest, GET, AGENTS_PATH + otherAgentId));
response = ex.getResponse();
assertNotFound(response);
map = responseToMap(response);
assertEquals("Failed to find agent with the provided agent id: " + otherAgentId, getErrorReasonFromResponseMap(map));

// Delete workflow from tenant without specifying to delete state
response = makeRequest(otherTenantRequest, DELETE, WORKFLOW_PATH + otherWorkflowId);
assertOK(response);
Expand Down
6 changes: 3 additions & 3 deletions src/test/java/org/opensearch/flowframework/workflow/DeployModelStepTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ public void testDeployModel() throws ExecutionException, InterruptedException, I

// Stub getTask for success case
doAnswer(invocation -> {
ActionListener<MLTask> actionListener = invocation.getArgument(1);
ActionListener<MLTask> actionListener = invocation.getArgument(2);
MLTask output = MLTask.builder().taskId(taskId).modelId(modelId).state(MLTaskState.COMPLETED).async(false).build();
actionListener.onResponse(output);
return null;
Expand Down Expand Up @@ -203,11 +203,11 @@ public void testDeployModelTaskFailure() throws IOException, InterruptedExceptio

// Stub getTask for success case
doAnswer(invocation -> {
ActionListener<MLTask> actionListener = invocation.getArgument(1);
ActionListener<MLTask> actionListener = invocation.getArgument(2);
MLTask output = MLTask.builder().taskId(taskId).modelId(modelId).state(MLTaskState.FAILED).async(false).error("error").build();
actionListener.onResponse(output);
return null;
}).when(machineLearningNodeClient).getTask(any(), any());
}).when(machineLearningNodeClient).getTask(any(), nullable(String.class), any());

PlainActionFuture<WorkflowData> future = this.deployModel.execute(
inputData.getNodeId(),
Expand Down
12 changes: 6 additions & 6 deletions src/test/java/org/opensearch/flowframework/workflow/RegisterLocalCustomModelStepTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ public void testRegisterLocalCustomModelSuccess() throws Exception {

// Stub getTask for success case
doAnswer(invocation -> {
ActionListener<MLTask> actionListener = invocation.getArgument(1);
ActionListener<MLTask> actionListener = invocation.getArgument(2);
MLTask output = MLTask.builder().taskId(taskId).modelId(modelId).state(MLTaskState.COMPLETED).async(false).build();
actionListener.onResponse(output);
return null;
Expand Down Expand Up @@ -209,7 +209,7 @@ public void testRegisterLocalCustomModelSuccess() throws Exception {
future.actionGet();

verify(machineLearningNodeClient, times(2)).register(any(MLRegisterModelInput.class), any());
verify(machineLearningNodeClient, times(2)).getTask(any(), any());
verify(machineLearningNodeClient, times(2)).getTask(any(), nullable(String.class), any());

assertEquals(modelId, future.get().getContent().get(MODEL_ID));
assertEquals(status, future.get().getContent().get(REGISTER_MODEL_STATUS));
Expand All @@ -231,11 +231,11 @@ public void testRegisterLocalCustomModelDeployStateUpdateFail() throws Exception

// Stub getTask for success case
doAnswer(invocation -> {
ActionListener<MLTask> actionListener = invocation.getArgument(1);
ActionListener<MLTask> actionListener = invocation.getArgument(2);
MLTask output = MLTask.builder().taskId(taskId).modelId(modelId).state(MLTaskState.COMPLETED).async(false).build();
actionListener.onResponse(output);
return null;
}).when(machineLearningNodeClient).getTask(any(), any());
}).when(machineLearningNodeClient).getTask(any(), nullable(String.class), any());

AtomicInteger invocationCount = new AtomicInteger(0);
doAnswer(invocation -> {
Expand Down Expand Up @@ -322,7 +322,7 @@ public void testRegisterLocalCustomModelTaskFailure() {

// Stub get ml task for failure case
doAnswer(invocation -> {
ActionListener<MLTask> actionListener = invocation.getArgument(1);
ActionListener<MLTask> actionListener = invocation.getArgument(2);
MLTask output = MLTask.builder()
.taskId(taskId)
.modelId(modelId)
Expand All @@ -332,7 +332,7 @@ public void testRegisterLocalCustomModelTaskFailure() {
.build();
actionListener.onResponse(output);
return null;
}).when(machineLearningNodeClient).getTask(any(), any());
}).when(machineLearningNodeClient).getTask(any(), nullable(String.class), any());

PlainActionFuture<WorkflowData> future = this.registerLocalModelStep.execute(
workflowData.getNodeId(),
Expand Down
8 changes: 4 additions & 4 deletions ...est/java/org/opensearch/flowframework/workflow/RegisterLocalPretrainedModelStepTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ public void testRegisterLocalPretrainedModelSuccess() throws Exception {

// Stub getTask for success case
doAnswer(invocation -> {
ActionListener<MLTask> actionListener = invocation.getArgument(1);
ActionListener<MLTask> actionListener = invocation.getArgument(2);
MLTask output = MLTask.builder().taskId(taskId).modelId(modelId).state(MLTaskState.COMPLETED).async(false).build();
actionListener.onResponse(output);
return null;
Expand Down Expand Up @@ -196,7 +196,7 @@ public void testRegisterLocalPretrainedModelSuccess() throws Exception {
future.actionGet();

verify(machineLearningNodeClient, times(2)).register(any(MLRegisterModelInput.class), any());
verify(machineLearningNodeClient, times(2)).getTask(any(), any());
verify(machineLearningNodeClient, times(2)).getTask(any(), nullable(String.class), any());

assertEquals(modelId, future.get().getContent().get(MODEL_ID));
assertEquals(status, future.get().getContent().get(REGISTER_MODEL_STATUS));
Expand Down Expand Up @@ -240,7 +240,7 @@ public void testRegisterLocalPretrainedModelTaskFailure() {

// Stub get ml task for failure case
doAnswer(invocation -> {
ActionListener<MLTask> actionListener = invocation.getArgument(1);
ActionListener<MLTask> actionListener = invocation.getArgument(2);
MLTask output = MLTask.builder()
.taskId(taskId)
.modelId(modelId)
Expand All @@ -250,7 +250,7 @@ public void testRegisterLocalPretrainedModelTaskFailure() {
.build();
actionListener.onResponse(output);
return null;
}).when(machineLearningNodeClient).getTask(any(), any());
}).when(machineLearningNodeClient).getTask(any(), nullable(String.class), any());

PlainActionFuture<WorkflowData> future = this.registerLocalPretrainedModelStep.execute(
workflowData.getNodeId(),
Expand Down
Loading
Loading