diff --git a/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java b/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java index 9c2d1464e..3b2a648df 100644 --- a/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java +++ b/src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java @@ -253,12 +253,20 @@ private void onAutoCorrelations(Detector detector, Finding finding, Map { try { log.error("[CORRELATIONS] Exception encountered while searching correlation rule index for finding id {}", finding.getId(), e); - getValidDocuments(detectorType, indices, List.of(), List.of(), autoCorrelations); + if (!autoCorrelations.isEmpty()) { + getValidDocuments(detectorType, indices, List.of(), List.of(), autoCorrelations); + } else { + correlateFindingAction.onOperation(); + } } catch (Exception ex) { onFailure(ex); } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java index 9f771d66e..cac9b0eef 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportCorrelateFindingAction.java @@ -47,6 +47,7 @@ import org.opensearch.securityanalytics.correlation.alert.CorrelationAlertService; import org.opensearch.securityanalytics.correlation.alert.notifications.NotificationService; import org.opensearch.securityanalytics.logtype.LogTypeService; +import org.opensearch.securityanalytics.model.CorrelationRule; import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; @@ -137,7 +138,11 @@ protected void doExecute(Task task, ActionRequest request, ActionListener { diff --git a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java index aa2f6a6a3..22b668753 100644 --- a/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/correlation/CorrelationEngineRestApiIT.java @@ -164,7 +164,6 @@ public void testListCorrelationsWorkflow() throws IOException, InterruptedExcept @SuppressWarnings("unchecked") public void testBasicCorrelationEngineWorkflowWithoutRules() throws IOException, InterruptedException { - updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "true"); LogIndices indices = createIndices(); String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); @@ -220,7 +219,8 @@ public void testBasicCorrelationEngineWorkflowWithoutRules() throws IOException, } return false; } catch (Exception ex) { - return false; + // because no findings are found + return true; } }, 2, TimeUnit.MINUTES @@ -679,34 +679,46 @@ public void testBasicCorrelationEngineWorkflowWithFieldBasedRulesOnMultipleLogTy ); } - public void testBasicCorrelationEngineWorkflowWithIndexPatterns() throws IOException, InterruptedException { + @SuppressWarnings("unchecked") + public void testBasicCorrelationEngineWorkflowWithoutRulesAndWithoutAutoCorrelations() throws IOException, InterruptedException { updateClusterSetting(SecurityAnalyticsSettings.ENABLE_AUTO_CORRELATIONS.getKey(), "false"); - - LogIndices indices = new LogIndices(); - createTestIndex("windows1", windowsIndexMapping()); - createTestIndex("windows2", windowsIndexMapping()); - indices.windowsIndex = "windows*"; - createTestIndex("vpc_flow1", vpcFlowMappings()); - createTestIndex("vpc_flow2", vpcFlowMappings()); - indices.vpcFlowsIndex = "vpc_flow*"; + LogIndices indices = createIndices(); String vpcFlowMonitorId = createVpcFlowDetector(indices.vpcFlowsIndex); + String adLdapMonitorId = createAdLdapDetector(indices.adLdapLogsIndex); String testWindowsMonitorId = createTestWindowsDetector(indices.windowsIndex); + String appLogsMonitorId = createAppLogsDetector(indices.appLogsIndex); + String s3MonitorId = createS3Detector(indices.s3AccessLogsIndex); - String ruleId = createNetworkToWindowsFilterQueryBasedRule(indices); - - indexDoc("windows2", "2", randomDoc()); - Response executeResponse = executeAlertingMonitor(testWindowsMonitorId, Collections.emptyMap()); + indexDoc(indices.adLdapLogsIndex, "22", randomAdLdapDoc()); + Response executeResponse = executeAlertingMonitor(adLdapMonitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(1, noOfSigmaRuleMatches); + + indexDoc(indices.windowsIndex, "2", randomDoc()); + executeResponse = executeAlertingMonitor(testWindowsMonitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(5, noOfSigmaRuleMatches); - indexDoc("vpc_flow1", "1", randomVpcFlowDoc()); + indexDoc(indices.appLogsIndex, "4", randomAppLogDoc()); + executeResponse = executeAlertingMonitor(appLogsMonitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(0, noOfSigmaRuleMatches); + + indexDoc(indices.s3AccessLogsIndex, "5", randomS3AccessLogDoc()); + executeResponse = executeAlertingMonitor(s3MonitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(0, noOfSigmaRuleMatches); + + indexDoc(indices.vpcFlowsIndex, "1", randomVpcFlowDoc()); executeResponse = executeAlertingMonitor(vpcFlowMonitorId, Collections.emptyMap()); executeResults = entityAsMap(executeResponse); noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); Assert.assertEquals(1, noOfSigmaRuleMatches); - Thread.sleep(5000); // Call GetFindings API Map params = new HashMap<>(); @@ -714,31 +726,14 @@ public void testBasicCorrelationEngineWorkflowWithIndexPatterns() throws IOExce Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); Map getFindingsBody = entityAsMap(getFindingsResponse); String finding = ((List>) getFindingsBody.get("findings")).get(0).get("id").toString(); + try { + List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); + fail(); + } catch (Exception e) { + assertTrue(e.getMessage().contains("no such index")); + } - OpenSearchRestTestCase.waitUntil( - () -> { - try { - List> correlatedFindings = searchCorrelatedFindings(finding, "test_windows", 300000L, 10); - if (correlatedFindings.size() == 1) { - Assert.assertTrue(true); - - Assert.assertTrue(correlatedFindings.get(0).get("rules") instanceof List); - for (var correlatedFinding: correlatedFindings) { - if (correlatedFinding.get("detector_type").equals("network")) { - Assert.assertEquals(1, ((List) correlatedFinding.get("rules")).size()); - Assert.assertTrue(((List) correlatedFinding.get("rules")).contains(ruleId)); - return true; - } - } - } - return false; - } catch (Exception ex) { - return false; - } - }, - 2, TimeUnit.MINUTES - ); } public void testBasicCorrelationEngineWorkflowWithFieldBasedRulesAndDynamicTimeWindow() throws IOException, InterruptedException { diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 81f5cacfa..365880851 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -715,81 +715,6 @@ public void testCreateMappings_withIndexPattern_existing_indexTemplate_update_su assertTrue(props.containsKey("destination.port")); } - public void testCreateMappings_withIndexPattern_differentMappings_indexTemplateCleanup_success() throws IOException, InterruptedException { - String indexName1 = "test_index_1"; - String indexName2 = "test_index_2"; - String indexPattern = "test_index*"; - - createIndex(indexName1, Settings.EMPTY, null); - createIndex(indexName2, Settings.EMPTY, null); - - // client().performRequest(new Request("POST", "_refresh")); - - // Insert sample docs - String sampleDoc1 = "{" + - " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.source_transport_port\":4444" + - "}"; - String sampleDoc2 = "{" + - " \"netflow.destination_transport_port\":1234," + - " \"netflow.destination_ipv4_address\":\"10.53.111.14\"" + - "}"; - indexDoc(indexName1, "1", sampleDoc1); - indexDoc(indexName2, "1", sampleDoc2); - - // client().performRequest(new Request("POST", "_refresh")); - - // Execute CreateMappingsAction to add alias mapping for index - createMappingsAPI(indexPattern, "netflow"); - - DetectorInput input = new DetectorInput("", List.of(indexPattern), List.of(), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); - String detectorId = createDetector(TestHelpers.randomDetectorWithInputs(List.of((input)))); - - refreshAllIndices(); - - List componentTemplates = getAllComponentTemplates(); - assertEquals(1, componentTemplates.size()); - List composableIndexTemplates = getAllComposableIndexTemplates(); - assertEquals(2, composableIndexTemplates.size()); - - deleteDetector(detectorId); - - // Wait for clusterState update to be published/applied - OpenSearchTestCase.waitUntil(() -> { - try { - List ct = getAllComponentTemplates(); - if (ct.size() == 0) { - return true; - } else { - return false; - } - } catch (IOException e) { - - } - return false; - }); - OpenSearchTestCase.waitUntil(() -> { - try { - List cct = getAllComposableIndexTemplates(); - if (cct.size() == 1) { - return true; - } else { - return false; - } - } catch (IOException e) { - - } - return false; - }); - - componentTemplates = getAllComponentTemplates(); - assertEquals(0, componentTemplates.size()); - composableIndexTemplates = getAllComposableIndexTemplates(); - assertEquals(1, composableIndexTemplates.size()); - } - public void testCreateMappings_withIndexPattern_indexTemplate_createAndUpdate_success() throws IOException { String indexName1 = "test_index_1"; String indexName2 = "test_index_2";