Bugfix/10.7.1/alerts stopping tag rules exception

CHANGELOG.md

@@ -1,5 +1,7 @@
 # UTMStack 10.7.2 Release Notes
+-- Implemented backend support for filtering compliance reports based on active integrations, optimizing query performance and data retrieval.
 
-## New Features and Improvements
--- Significant improvement in CPU performance
--- Improved memory distribution
+### Bug Fixes
+-- Improved exception handling in `automaticReview` to prevent the process from stopping due to errors, ensuring the system continues evaluating alerts even if a specific rule fails.
+-- Improved operator selection for more accurate and consistent filtering.
+-- Introduced new compliance reports aligned with the PCI DSS standard to expand auditing capabilities.

backend/src/main/java/com/park/utmstack/domain/chart_builder/UtmDashboard.java

@@ -9,6 +9,8 @@
 import javax.validation.constraints.Size;
 import java.io.Serializable;
 import java.time.Instant;
+import java.util.HashSet;
+import java.util.Set;
 
 /**
  * A UtmDashboard.
@@ -63,6 +65,14 @@ public class UtmDashboard implements Serializable {
     @Column(name = "system_owner")
     private Boolean systemOwner;
 
+    @ManyToMany
+    @JoinTable(
+            name = "utm_dashboard_visualization",
+            joinColumns = @JoinColumn(name = "id_dashboard"),
+            inverseJoinColumns = @JoinColumn(name = "id_visualization")
+    )
+    private Set<UtmVisualization> visualizations = new HashSet<>();
+
     public Long getId() {
         return id;
     }

backend/src/main/java/com/park/utmstack/domain/chart_builder/UtmVisualization.java

@@ -18,7 +18,9 @@
 import javax.validation.constraints.Size;
 import java.io.Serializable;
 import java.time.Instant;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 /**
  * A UtmVisualization.
@@ -106,6 +108,9 @@ public class UtmVisualization implements Serializable {
     @JoinColumn(name = "id_pattern", referencedColumnName = "id", insertable = false, updatable = false)
     private UtmIndexPattern pattern;
 
+    @ManyToMany(mappedBy = "visualizations")
+    private Set<UtmDashboard> dashboards = new HashSet<>();
+
     public Long getId() {
         return id;
     }

backend/src/main/java/com/park/utmstack/repository/compliance/UtmComplianceReportConfigRepository.java

@@ -30,13 +30,15 @@ public interface UtmComplianceReportConfigRepository extends JpaRepository<UtmCo
     @Query("delete from UtmComplianceReportConfig r where r.standardSectionId in (select s.id from UtmComplianceStandardSection s where s.standardId = :standardId) and r.id not in :reportIds")
     void deleteReportsByStandardIdAndIdNotIn(@Param("standardId") Long standardId, @Param("reportIds") List<Long> reportIds);
 
-    @Query(value = "SELECT cfg FROM UtmComplianceReportConfig cfg " +
+    @Query(value = "SELECT DISTINCT cfg FROM UtmComplianceReportConfig cfg " +
             "JOIN cfg.section sec " +
             "LEFT JOIN cfg.associatedDashboard d " +
+            "LEFT JOIN d.visualizations v " +
             "WHERE (:standardId IS NULL OR sec.standardId = :standardId) " +
             "AND (:solution IS NULL OR lower(cfg.configSolution) LIKE %:solution%) " +
             "AND (:sectionId IS NULL OR sec.id = :sectionId) " +
-            "AND (:search IS NULL OR lower(cfg.configReportName) LIKE %:search% OR d.name LIKE %:search%)")
+            "AND (:search IS NULL OR lower(cfg.configReportName) LIKE %:search% OR d.name LIKE %:search%) " +
+            "AND (v.pattern.isActive = true)")
     Page<UtmComplianceReportConfig> getReportsByFilters(
             @Param("standardId") Long standardId,
             @Param("solution") String solution,

backend/src/main/java/com/park/utmstack/service/UtmAlertTagRuleService.java

@@ -104,7 +104,7 @@ public Page<UtmAlertTagRule> findByFilter(AlertTagRuleFilterVM filters, Pageable
             TypedParameterValue isDeleted = new TypedParameterValue(new BooleanType(), filters.getRuleDeleted());
 
             return alertTagRuleRepository.findByFilter(id, name, conditionField,
-                conditionValue, tagIds, isActive, isDeleted, pageable);
+                    conditionValue, tagIds, isActive, isDeleted, pageable);
         } catch (Exception e) {
             throw new Exception(ctx + ": " + e.getMessage());
         }
@@ -145,7 +145,7 @@ public void automaticReview() {
         final String ctx = CLASSNAME + ".automaticReview";
         try {
             // If no new alerts have been received, stop execution
-            if (alertUtil.countAlertsByStatus(AlertStatus.AUTOMATIC_REVIEW.getCode()) == 0)
+            if (alertUtil.countAlertsByStatus(AlertStatus.OPEN.getCode()) == 0)
                 return;
 
             // Getting all registered rules
@@ -154,12 +154,12 @@ public void automaticReview() {
             // Getting rules that are actives and are not deleted
             if (!CollectionUtils.isEmpty(tagRules))
                 tagRules = tagRules.stream()
-                    .filter(rule -> rule.getRuleActive() && !rule.getRuleDeleted())
-                    .collect(Collectors.toList());
+                        .filter(rule -> rule.getRuleActive() && !rule.getRuleDeleted())
+                        .collect(Collectors.toList());
 
             // Getting rule evaluation start time
             Instant rulesEvaluationStart = LocalDateTime.now().toInstant(ZoneOffset.UTC)
-                .truncatedTo(ChronoUnit.SECONDS);
+                    .truncatedTo(ChronoUnit.SECONDS);
 
             // If there is any rule
             if (!CollectionUtils.isEmpty(tagRules))
@@ -174,17 +174,17 @@ public void automaticReview() {
         }
     }
 
-    private void releaseToOpen(Instant rulesEvaluationStart) throws Exception {
+    private void releaseToOpen(Instant rulesEvaluationStart) {
         final String ctx = CLASSNAME + ".releaseToOpen";
         try {
             String ruleScript = "ctx._source.status=%1$s;" +
-                "ctx._source.statusLabel='%2$s';" +
-                "ctx._source.statusObservation='%3$s';";
+                    "ctx._source.statusLabel='%2$s';" +
+                    "ctx._source.statusObservation='%3$s';";
 
             String statusObservation = "This alert has been evaluated by the tag rules engine";
 
             String script = String.format(ruleScript, AlertStatus.OPEN.getCode(), AlertStatus.OPEN.getName(),
-                statusObservation);
+                    statusObservation);
 
             List<FilterType> filters = new ArrayList<>();
             filters.add(new FilterType(Constants.alertStatus, OperatorType.IS, AlertStatus.AUTOMATIC_REVIEW.getCode()));
@@ -196,51 +196,55 @@ private void releaseToOpen(Instant rulesEvaluationStart) throws Exception {
             elasticsearchService.updateByQuery(query, indexPattern, script);
 
             alertPointcut.automaticAlertStatusChangePointcut(query, AlertStatus.OPEN.getCode(),
-                statusObservation, indexPattern);
+                    statusObservation, indexPattern);
         } catch (Exception e) {
-            throw new Exception(ctx + ": " + e.getMessage());
+            String msg = ctx + ": " + e.getMessage();
+            eventService.createEvent(msg, ApplicationEventType.ERROR);
+            log.error(msg, e.getMessage(), e);
         }
     }
 
-    private void applyTagRule(List<UtmAlertTagRule> rules, Instant rulesEvaluationStart) throws Exception {
+    private void applyTagRule(List<UtmAlertTagRule> rules, Instant rulesEvaluationStart) {
         final String ctx = CLASSNAME + ".applyTagRule";
-        try {
-            for (UtmAlertTagRule rule : rules) {
+        for (UtmAlertTagRule rule : rules) {
+            try {
                 List<FilterType> filters = rule.getConditions();
                 filters.add(new FilterType(Constants.alertStatus, OperatorType.IS, AlertStatus.AUTOMATIC_REVIEW.getCode()));
                 filters.add(new FilterType(Constants.timestamp, OperatorType.IS_LESS_THAN_OR_EQUALS, rulesEvaluationStart.toString()));
 
                 Query query = SearchUtil.toQuery(filters);
 
                 String script = "if (!ctx._source.containsKey('tags') || ctx._source.tags == null || ctx._source.tags.empty)\n" +
-                    "\tctx._source.tags = new ArrayList();\n" +
-                    "ctx._source.tags.addAll([%1$s]);\n" +
-                    "ctx._source.tags = ctx._source.tags.stream().distinct().collect(Collectors.toList());\n" +
-                    "\n" +
-                    "if (!ctx._source.containsKey('TagRulesApplied') || ctx._source.TagRulesApplied == null || ctx._source.TagRulesApplied.empty)\n" +
-                    "\tctx._source.TagRulesApplied = new ArrayList();\n" +
-                    "ctx._source.TagRulesApplied.add(%2$s);\n" +
-                    "ctx._source.TagRulesApplied = ctx._source.TagRulesApplied.stream().distinct().collect(Collectors.toList());" +
-                    "\n" +
-                    "if (ctx._source.tags.contains('False positive')) {\n" +
-                    "\tctx._source.status=%3$s;\n" +
-                    "\tctx._source.statusLabel='%4$s';\n" +
-                    "\tctx._source.statusObservation='Status changed to completed because alert was tagged as False positive';\n" +
-                    "\n}";
+                        "\tctx._source.tags = new ArrayList();\n" +
+                        "ctx._source.tags.addAll([%1$s]);\n" +
+                        "ctx._source.tags = ctx._source.tags.stream().distinct().collect(Collectors.toList());\n" +
+                        "\n" +
+                        "if (!ctx._source.containsKey('TagRulesApplied') || ctx._source.TagRulesApplied == null || ctx._source.TagRulesApplied.empty)\n" +
+                        "\tctx._source.TagRulesApplied = new ArrayList();\n" +
+                        "ctx._source.TagRulesApplied.add(%2$s);\n" +
+                        "ctx._source.TagRulesApplied = ctx._source.TagRulesApplied.stream().distinct().collect(Collectors.toList());" +
+                        "\n" +
+                        "if (ctx._source.tags.contains('False positive')) {\n" +
+                        "\tctx._source.status=%3$s;\n" +
+                        "\tctx._source.statusLabel='%4$s';\n" +
+                        "\tctx._source.statusObservation='Status changed to completed because alert was tagged as False positive';\n" +
+                        "\n}";
 
                 List<UtmAlertTag> tags = alertTagService.findAllByIdIn(rule.getAppliedTagsAsListOfLong());
                 String indexPattern = Constants.SYS_INDEX_PATTERN.get(SystemIndexPattern.ALERTS);
 
                 String tagsForInsert = tags.stream().map(tag -> "'" + tag.getTagName() + "'").collect(Collectors.joining(","));
 
                 elasticsearchService.updateByQuery(query, indexPattern, String.format(script, tagsForInsert, rule.getId(),
-                    AlertStatus.COMPLETED.getCode(), AlertStatus.COMPLETED.getName()));
+                        AlertStatus.COMPLETED.getCode(), AlertStatus.COMPLETED.getName()));
 
                 String tagsForLogs = tags.stream().map(UtmAlertTag::getTagName).collect(Collectors.joining(","));
                 alertPointcut.automaticAlertTagsChangePointcut(query, tagsForLogs, rule.getRuleName(), indexPattern);
+            } catch (Exception e) {
+                String msg = String.format("%s: Error applying rule '%s' - %s", ctx, rule.getRuleName(), e.getMessage());
+                eventService.createEvent(msg, ApplicationEventType.ERROR);
+                log.error(msg, e.getMessage(), e);
             }
-        } catch (Exception e) {
-            throw new Exception(ctx + ": " + e.getMessage());
         }
     }
 }

backend/src/main/java/com/park/utmstack/service/compliance/config/UtmComplianceReportConfigService.java

@@ -9,18 +9,21 @@
 import com.park.utmstack.domain.compliance.UtmComplianceStandard;
 import com.park.utmstack.domain.compliance.UtmComplianceStandardSection;
 import com.park.utmstack.domain.compliance.enums.ComplianceStatus;
+import com.park.utmstack.domain.index_pattern.UtmIndexPattern;
 import com.park.utmstack.repository.compliance.UtmComplianceReportConfigRepository;
 import com.park.utmstack.service.chart_builder.UtmDashboardService;
 import com.park.utmstack.service.chart_builder.UtmDashboardVisualizationService;
 import com.park.utmstack.service.elasticsearch.ElasticsearchService;
 import com.park.utmstack.service.elasticsearch.SearchUtil;
+import com.park.utmstack.service.index_pattern.UtmIndexPatternService;
 import com.park.utmstack.util.UtilPagination;
 import com.park.utmstack.util.chart_builder.elasticsearch_dsl.requests.RequestDsl;
 import com.park.utmstack.util.exceptions.UtmElasticsearchException;
 import org.opensearch.client.opensearch.core.SearchResponse;
 import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageImpl;
+import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
@@ -32,6 +35,7 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 @Service
 public class UtmComplianceReportConfigService {
@@ -52,7 +56,7 @@ public UtmComplianceReportConfigService(UtmComplianceReportConfigRepository comp
                                             UtmComplianceStandardService standardService,
                                             UtmComplianceStandardSectionService standardSectionService,
                                             UtmDashboardVisualizationService dashboardVisualizationService, ElasticsearchService elasticsearchService,
-                                            EntityManager em) {
+                                            EntityManager em, UtmIndexPatternService indexPatternService) {
         this.complianceReportConfigRepository = complianceReportConfigRepository;
         this.dashboardService = dashboardService;
         this.standardService = standardService;

backend/src/main/java/com/park/utmstack/web/rest/UtmAlertTagRuleResource.java

@@ -1,5 +1,6 @@
 package com.park.utmstack.web.rest;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.park.utmstack.domain.UtmAlertTag;
 import com.park.utmstack.domain.UtmAlertTagRule;
 import com.park.utmstack.domain.application_events.enums.ApplicationEventType;
@@ -26,6 +27,7 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.stream.Collectors;
 
 /**
  * REST controller for managing UtmTagRule.
@@ -41,12 +43,16 @@ public class UtmAlertTagRuleResource {
     private final ApplicationEventService applicationEventService;
     private final UtmAlertTagService alertTagService;
 
+    private final ObjectMapper objectMapper;
+
+
     public UtmAlertTagRuleResource(UtmAlertTagRuleService tagRuleService,
                                    ApplicationEventService applicationEventService,
-                                   UtmAlertTagService alertTagService) {
+                                   UtmAlertTagService alertTagService, ObjectMapper objectMapper) {
         this.tagRuleService = tagRuleService;
         this.applicationEventService = applicationEventService;
         this.alertTagService = alertTagService;
+        this.objectMapper = objectMapper;
     }
 
     @PostMapping("/alert-tag-rules")
@@ -80,13 +86,16 @@ public ResponseEntity<AlertTagRuleVM> createAlertTagRule(@Valid @RequestBody Ale
     public ResponseEntity<AlertTagRuleVM> updateAlertTagRule(@Valid @RequestBody AlertTagRuleVM ruleVM) throws URISyntaxException {
         final String ctx = CLASSNAME + ".updateAlertTagRule";
         try {
-            if (Objects.isNull(ruleVM.getId()))
-                throw new Exception("ID can't be null");
+              UtmAlertTagRule utmAlertTagRule = this.tagRuleService.findOne(ruleVM.getId())
+                      .orElseThrow(() -> new RuntimeException("Tag rule with" + ruleVM.getId() + "not found"));
 
-            UtmAlertTagRule alertTagRule = tagRuleService.save(new UtmAlertTagRule(ruleVM));
+              utmAlertTagRule.setRuleConditions(this.objectMapper.writeValueAsString(ruleVM.getConditions()));
+              utmAlertTagRule.setRuleDescription(ruleVM.getDescription());
+              utmAlertTagRule.setRuleName(ruleVM.getName());
+              utmAlertTagRule.setRuleAppliedTags(ruleVM.getTags().stream().map(tag -> String.valueOf(tag.getId()))
+                      .collect(Collectors.joining(",")));
 
-            ruleVM.setLastModifiedBy(alertTagRule.getLastModifiedBy());
-            ruleVM.setLastModifiedDate(alertTagRule.getLastModifiedDate());
+              this.tagRuleService.save(utmAlertTagRule);
 
             return ResponseEntity.ok(ruleVM);
         } catch (Exception e) {