SLCORE-1819 Migrate existing known findings to the new database

Author: damien-urruty-sonarsource

backend/commons/src/main/java/org/sonarsource/sonarlint/core/commons/storage/repository/Findings.java

@@ -0,0 +1,34 @@
+/*
+ * SonarLint Core - Commons
+ * Copyright (C) 2016-2025 SonarSource Sàrl
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonarsource.sonarlint.core.commons.storage.repository;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.sonarsource.sonarlint.core.commons.KnownFinding;
+
+public record Findings(List<KnownFinding> issues, List<KnownFinding> hotspots) {
+  public Findings mergeWith(Findings other) {
+    var mergedIssues = new ArrayList<>(issues);
+    mergedIssues.addAll(other.issues);
+    var mergedHotspots = new ArrayList<KnownFinding>(hotspots);
+    mergedHotspots.addAll(other.hotspots);
+    return new Findings(mergedIssues, mergedHotspots);
+  }
+}

backend/commons/src/main/java/org/sonarsource/sonarlint/core/commons/storage/repository/KnownFindingsRepository.java

@@ -21,15 +21,18 @@
 
 import java.nio.file.Path;
 import java.time.LocalDateTime;
-import java.time.ZoneId;
+import java.time.ZoneOffset;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
 import org.jooq.Configuration;
 import org.jooq.Record;
 import org.sonarsource.sonarlint.core.commons.KnownFinding;
 import org.sonarsource.sonarlint.core.commons.KnownFindingType;
 import org.sonarsource.sonarlint.core.commons.LineWithHash;
 import org.sonarsource.sonarlint.core.commons.api.TextRangeWithHash;
 import org.sonarsource.sonarlint.core.commons.storage.SonarLintDatabase;
+import org.sonarsource.sonarlint.core.commons.storage.model.tables.records.KnownFindingsRecord;
 
 import static org.sonarsource.sonarlint.core.commons.storage.model.Tables.KNOWN_FINDINGS;
 
@@ -41,6 +44,47 @@ public KnownFindingsRepository(SonarLintDatabase database) {
     this.database = database;
   }
 
+  public void storeFindings(Map<String, Map<Path, Findings>> findingsPerFilePerConfigScopeId) {
+    database.dsl().batchMerge(findingsPerFilePerConfigScopeId.entrySet().stream()
+      .flatMap(configScopeEntry -> {
+        var configScopeId = configScopeEntry.getKey();
+        return configScopeEntry.getValue().entrySet().stream()
+          .flatMap(fileEntry -> {
+            var filePath = fileEntry.getKey();
+            var findings = fileEntry.getValue();
+            return Stream.<KnownFindingsRecord>concat(
+              findings.issues().stream()
+                .map(finding -> createRecord(finding, configScopeId, filePath, KnownFindingType.ISSUE)),
+              findings.hotspots().stream()
+                .map(finding -> createRecord(finding, configScopeId, filePath, KnownFindingType.HOTSPOT)));
+          });
+      })
+      .toList())
+      .execute();
+  }
+
+  private static KnownFindingsRecord createRecord(KnownFinding finding, String configScopeId, Path filePath, KnownFindingType type) {
+    var textRangeWithHash = finding.getTextRangeWithHash();
+    var lineWithHash = finding.getLineWithHash();
+    var introductionDate = LocalDateTime.ofInstant(finding.getIntroductionDate(), ZoneOffset.UTC);
+    return new KnownFindingsRecord(
+      finding.getId(),
+      configScopeId,
+      filePath.toString(),
+      finding.getServerKey(),
+      finding.getRuleKey(),
+      finding.getMessage(),
+      introductionDate,
+      type.name(),
+      textRangeWithHash == null ? null : textRangeWithHash.getStartLine(),
+      textRangeWithHash == null ? null : textRangeWithHash.getStartLineOffset(),
+      textRangeWithHash == null ? null : textRangeWithHash.getEndLine(),
+      textRangeWithHash == null ? null : textRangeWithHash.getEndLineOffset(),
+      textRangeWithHash == null ? null : textRangeWithHash.getHash(),
+      lineWithHash == null ? null : lineWithHash.getNumber(),
+      lineWithHash == null ? null : lineWithHash.getHash());
+  }
+
   public void storeKnownIssues(String configurationScopeId, Path clientRelativePath, List<KnownFinding> newKnownIssues) {
     storeKnownFindings(configurationScopeId, clientRelativePath, newKnownIssues, KnownFindingType.ISSUE);
   }
@@ -69,7 +113,7 @@ private void storeKnownFindings(String configurationScopeId, Path clientRelative
       var lineWithHash = finding.getLineWithHash();
       var line = lineWithHash == null ? null : lineWithHash.getNumber();
       var lineHash = lineWithHash == null ? null : lineWithHash.getHash();
-      var introDate = LocalDateTime.ofInstant(finding.getIntroductionDate(), ZoneId.systemDefault());
+      var introDate = LocalDateTime.ofInstant(finding.getIntroductionDate(), ZoneOffset.UTC);
       trx.dsl().mergeInto(KNOWN_FINDINGS)
         .using(trx.dsl().selectOne())
         .on(KNOWN_FINDINGS.ID.eq(finding.getId()))
@@ -95,8 +139,7 @@ private void storeKnownFindings(String configurationScopeId, Path clientRelative
         .values(finding.getId(), configurationScopeId, clientRelativePath.toString(), finding.getServerKey(), finding.getRuleKey(),
           finding.getMessage(), introDate, type.name(),
           startLine, startLineOffset, endLine, endLineOffset, textRangeHash,
-          line, lineHash
-        )
+          line, lineHash)
         .execute();
     }));
   }
@@ -106,8 +149,7 @@ private List<KnownFinding> getKnownFindingsForFile(String configurationScopeId,
       .selectFrom(KNOWN_FINDINGS)
       .where(KNOWN_FINDINGS.CONFIGURATION_SCOPE_ID.eq(configurationScopeId)
         .and(KNOWN_FINDINGS.IDE_RELATIVE_FILE_PATH.eq(filePath.toString()))
-        .and(KNOWN_FINDINGS.FINDING_TYPE.eq(type.name()))
-      )
+        .and(KNOWN_FINDINGS.FINDING_TYPE.eq(type.name())))
       .fetch();
     return issuesInFile.stream()
       .map(KnownFindingsRepository::recordToKnownFinding)
@@ -116,7 +158,7 @@ private List<KnownFinding> getKnownFindingsForFile(String configurationScopeId,
 
   private static KnownFinding recordToKnownFinding(Record rec) {
     var id = rec.get(KNOWN_FINDINGS.ID);
-    var introductionDate = rec.get(KNOWN_FINDINGS.INTRODUCTION_DATE).atZone(ZoneId.systemDefault()).toInstant();
+    var introductionDate = rec.get(KNOWN_FINDINGS.INTRODUCTION_DATE).toInstant(ZoneOffset.UTC);
     var textRangeWithHash = getTextRangeWithHash(rec);
     var lineWithHash = getLineWithHash(rec);
     return new KnownFinding(
@@ -125,25 +167,27 @@ private static KnownFinding recordToKnownFinding(Record rec) {
       textRangeWithHash, lineWithHash,
       rec.get(KNOWN_FINDINGS.RULE_KEY),
       rec.get(KNOWN_FINDINGS.MESSAGE),
-      introductionDate
-    );
+      introductionDate);
   }
 
   private static LineWithHash getLineWithHash(Record rec) {
-    if (rec.get(KNOWN_FINDINGS.LINE) == null) return null;
     var line = rec.get(KNOWN_FINDINGS.LINE);
+    if (line == null) {
+      return null;
+    }
     var hash = rec.get(KNOWN_FINDINGS.LINE_HASH);
     return new LineWithHash(line, hash);
   }
 
   private static TextRangeWithHash getTextRangeWithHash(Record rec) {
-    if (rec.get(KNOWN_FINDINGS.START_LINE) == null) return null;
     var startLine = rec.get(KNOWN_FINDINGS.START_LINE);
+    if (startLine == null) {
+      return null;
+    }
     var endLine = rec.get(KNOWN_FINDINGS.END_LINE);
     var startLineOffset = rec.get(KNOWN_FINDINGS.START_LINE_OFFSET);
     var endLineOffset = rec.get(KNOWN_FINDINGS.END_LINE_OFFSET);
     var hash = rec.get(KNOWN_FINDINGS.TEXT_RANGE_HASH);
     return new TextRangeWithHash(startLine, startLineOffset, endLine, endLineOffset, hash);
   }
-
 }

backend/core/src/main/java/org/sonarsource/sonarlint/core/tracking/KnownFindingsStorageService.java

@@ -20,11 +20,17 @@
 package org.sonarsource.sonarlint.core.tracking;
 
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.annotation.PreDestroy;
+import org.apache.commons.io.FileUtils;
 import org.sonarsource.sonarlint.core.UserPaths;
 
+import static org.sonarsource.sonarlint.core.commons.storage.XodusPurgeUtils.deleteInFolderWithPattern;
+import static org.sonarsource.sonarlint.core.tracking.XodusKnownFindingsStore.BACKUP_TAR_GZ;
+import static org.sonarsource.sonarlint.core.tracking.XodusKnownFindingsStore.KNOWN_FINDINGS_STORE;
+
 public class KnownFindingsStorageService {
 
   private final Path projectsStorageBaseDir;
@@ -36,6 +42,10 @@ public KnownFindingsStorageService(UserPaths userPaths) {
     this.workDir = userPaths.getWorkDir();
   }
 
+  public boolean exists() {
+    return Files.exists(projectsStorageBaseDir.resolve(XodusKnownFindingsStore.BACKUP_TAR_GZ));
+  }
+
   public synchronized XodusKnownFindingsStore get() {
     var store = trackedIssuesStore.get();
     if (store == null) {
@@ -54,8 +64,17 @@ public synchronized XodusKnownFindingsStore get() {
   public void close() {
     var store = trackedIssuesStore.get();
     if (store != null) {
-      store.close();
+      store.backupAndClose();
     }
   }
 
+  public void delete() {
+    var store = trackedIssuesStore.getAndSet(null);
+    if (store != null) {
+      store.close();
+    }
+    FileUtils.deleteQuietly(projectsStorageBaseDir.resolve(BACKUP_TAR_GZ).toFile());
+    deleteInFolderWithPattern(workDir, KNOWN_FINDINGS_STORE + "*");
+    deleteInFolderWithPattern(projectsStorageBaseDir, "known_findings_backup*");
+  }
 }

backend/core/src/main/java/org/sonarsource/sonarlint/core/tracking/TrackingService.java

@@ -19,6 +19,7 @@
  */
 package org.sonarsource.sonarlint.core.tracking;
 
+import jakarta.annotation.PostConstruct;
 import java.net.URI;
 import java.nio.file.Path;
 import java.time.Instant;
@@ -117,6 +118,24 @@ public TrackingService(SonarLintRpcClient client, ConfigurationRepository config
     this.databaseService = databaseService;
   }
 
+  @PostConstruct
+  public void migrateData() {
+    if (dogfoodEnvironmentDetectionService.isDogfoodEnvironment()) {
+      if (knownFindingsStorageService.exists()) {
+        try {
+          var repository = new KnownFindingsRepository(databaseService.getDatabase());
+          var xodusKnownFindingsStore = knownFindingsStorageService.get();
+          var findingsPerConfigScope = xodusKnownFindingsStore.loadAll();
+          repository.storeFindings(findingsPerConfigScope);
+        } catch (Exception e) {
+          LOG.error("Unable to migrate known findings, will use fresh DB", e);
+        }
+      }
+      // always call to remove lingering temporary files
+      knownFindingsStorageService.delete();
+    }
+  }
+
   @EventListener
   public void onAnalysisStarted(AnalysisStartedEvent event) {
     var configurationScopeId = event.getConfigurationScopeId();
@@ -168,7 +187,6 @@ private MatchingResult matchWithServerFindings(String configurationScopeId, Matc
     var effectiveBindingOpt = configurationRepository.getEffectiveBinding(configurationScopeId);
     var activeBranchOpt = branchTrackingService.awaitEffectiveSonarProjectBranch(configurationScopeId);
     var translationOpt = pathTranslationService.getOrComputePathTranslation(configurationScopeId);
-    var knownFindingsStore = knownFindingsStorageService.get();
     var issuesToReport = matchingSession.getIssuesPerFile();
     var hotspotsToReport = matchingSession.getSecurityHotspotsPerFile();
     if (effectiveBindingOpt.isPresent() && activeBranchOpt.isPresent() && translationOpt.isPresent()) {
@@ -191,34 +209,34 @@ private MatchingResult matchWithServerFindings(String configurationScopeId, Matc
         return Map.entry(ideRelativePath, matches);
       }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
     }
-    issuesToReport.forEach((clientRelativePath, trackedIssues) -> storeTrackedIssues(knownFindingsStore, configurationScopeId, clientRelativePath, trackedIssues));
-    hotspotsToReport.forEach((clientRelativePath, trackedHotspots) -> storeTrackedSecurityHotspots(knownFindingsStore, configurationScopeId, clientRelativePath, trackedHotspots));
+    issuesToReport.forEach((clientRelativePath, trackedIssues) -> storeTrackedIssues(configurationScopeId, clientRelativePath, trackedIssues));
+    hotspotsToReport.forEach((clientRelativePath, trackedHotspots) -> storeTrackedSecurityHotspots(configurationScopeId, clientRelativePath, trackedHotspots));
     eventPublisher.publishEvent(new MatchingSessionEndedEvent(matchingSession.countNewIssues(), matchingSession.countRemainingUnmatchedIssues()));
     return new MatchingResult(issuesToReport, hotspotsToReport);
   }
 
-  private void storeTrackedIssues(XodusKnownFindingsStore knownIssuesStore, String configurationScopeId, Path clientRelativePath, Collection<TrackedIssue> newKnownIssues) {
+  private void storeTrackedIssues(String configurationScopeId, Path clientRelativePath, Collection<TrackedIssue> newKnownIssues) {
     if (dogfoodEnvironmentDetectionService.isDogfoodEnvironment()) {
       var knownFindingsRepository = new KnownFindingsRepository(databaseService.getDatabase());
       knownFindingsRepository.storeKnownIssues(configurationScopeId, clientRelativePath,
         newKnownIssues.stream().map(i -> new KnownFinding(i.getId(), i.getServerKey(), i.getTextRangeWithHash(), i.getLineWithHash(), i.getRuleKey(), i.getMessage(),
           i.getIntroductionDate())).toList());
     } else {
-      knownIssuesStore.storeKnownIssues(configurationScopeId, clientRelativePath,
+      knownFindingsStorageService.get().storeKnownIssues(configurationScopeId, clientRelativePath,
         newKnownIssues.stream().map(i -> new KnownFinding(i.getId(), i.getServerKey(), i.getTextRangeWithHash(), i.getLineWithHash(), i.getRuleKey(), i.getMessage(),
           i.getIntroductionDate())).toList());
     }
   }
 
-  private void storeTrackedSecurityHotspots(XodusKnownFindingsStore knownIssuesStore, String configurationScopeId, Path clientRelativePath,
+  private void storeTrackedSecurityHotspots(String configurationScopeId, Path clientRelativePath,
     Collection<TrackedIssue> newKnownSecurityHotspots) {
     if (dogfoodEnvironmentDetectionService.isDogfoodEnvironment()) {
       var knownFindingsRepository = new KnownFindingsRepository(databaseService.getDatabase());
       knownFindingsRepository.storeKnownSecurityHotspots(configurationScopeId, clientRelativePath,
         newKnownSecurityHotspots.stream().map(i -> new KnownFinding(i.getId(), i.getServerKey(), i.getTextRangeWithHash(), i.getLineWithHash(), i.getRuleKey(), i.getMessage(),
           i.getIntroductionDate())).toList());
     } else {
-      knownIssuesStore.storeKnownSecurityHotspots(configurationScopeId, clientRelativePath,
+      knownFindingsStorageService.get().storeKnownSecurityHotspots(configurationScopeId, clientRelativePath,
         newKnownSecurityHotspots.stream().map(i -> new KnownFinding(i.getId(), i.getServerKey(), i.getTextRangeWithHash(), i.getLineWithHash(), i.getRuleKey(), i.getMessage(),
           i.getIntroductionDate())).toList());
     }
@@ -283,7 +301,7 @@ private static TrackedIssue updateRawHotspotWithServerData(TrackedIssue trackedH
       serverHotspot.getStatus().isResolved(), trackedHotspot.getSeverity(), RuleType.SECURITY_HOTSPOT, trackedHotspot.getRuleKey(),
       trackedHotspot.getTextRangeWithHash(), trackedHotspot.getLineWithHash(),
       serverHotspot.getKey(), trackedHotspot.getImpacts(), trackedHotspot.getFlows(), trackedHotspot.getQuickFixes(),
-      serverHotspot.getVulnerabilityProbability(), HotspotStatus.valueOf(serverHotspot.getStatus().name()), null,  trackedHotspot.getRuleDescriptionContextKey(),
+      serverHotspot.getVulnerabilityProbability(), HotspotStatus.valueOf(serverHotspot.getStatus().name()), null, trackedHotspot.getRuleDescriptionContextKey(),
       trackedHotspot.getCleanCodeAttribute(), trackedHotspot.getFileUri());
   }
 
@@ -301,7 +319,6 @@ private static TrackedIssue updateTrackedIssueWithLocalOnlyIssueData(TrackedIssu
   }
 
   private MatchingSession startMatchingSession(String configurationScopeId, Set<Path> fileRelativePaths, Set<URI> fileUris, UnaryOperator<String> fileContentProvider) {
-    var knownFindingsStore = knownFindingsStorageService.get();
     var dogfoodEnvironment = dogfoodEnvironmentDetectionService.isDogfoodEnvironment();
     Map<Path, List<KnownFinding>> issuesByRelativePath;
     Map<Path, List<KnownFinding>> hotspotsByRelativePath;
@@ -312,6 +329,7 @@ private MatchingSession startMatchingSession(String configurationScopeId, Set<Pa
       hotspotsByRelativePath = fileRelativePaths.stream()
         .collect(toMap(Function.identity(), relativePath -> knownFindingsRepository.loadSecurityHotspotsForFile(configurationScopeId, relativePath)));
     } else {
+      var knownFindingsStore = knownFindingsStorageService.get();
       issuesByRelativePath = fileRelativePaths.stream()
         .collect(toMap(Function.identity(), relativePath -> knownFindingsStore.loadIssuesForFile(configurationScopeId, relativePath)));
       hotspotsByRelativePath = fileRelativePaths.stream()