Skip to content

Commit ff9ef6b

Browse files
damien-urruty-sonarsourcedamien-urruty-sonarsource
committed
SLCORE-1819 Migrate existing known findings to the new database
1 parent d8ccc6e commit ff9ef6b

File tree

9 files changed

+233
-41
lines changed

9 files changed

+233
-41
lines changed

‎backend/commons/src/main/java/org/sonarsource/sonarlint/core/commons/storage/repository/Findings.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
* SonarLint Core - Commons
3+
* Copyright (C) 2016-2025 SonarSource Sàrl
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonarsource.sonarlint.core.commons.storage.repository;
21+
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
import org.sonarsource.sonarlint.core.commons.KnownFinding;
25+
26+
public record Findings(List<KnownFinding> issues, List<KnownFinding> hotspots) {
27+
public Findings mergeWith(Findings other) {
28+
var mergedIssues = new ArrayList<>(issues);
29+
mergedIssues.addAll(other.issues);
30+
var mergedHotspots = new ArrayList<KnownFinding>(hotspots);
31+
mergedHotspots.addAll(other.hotspots);
32+
return new Findings(mergedIssues, mergedHotspots);
33+
}
34+
}

‎backend/commons/src/main/java/org/sonarsource/sonarlint/core/commons/storage/repository/KnownFindingsRepository.java‎

Lines changed: 56 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,18 @@
2121

2222
import java.nio.file.Path;
2323
import java.time.LocalDateTime;
24-
import java.time.ZoneId;
24+
import java.time.ZoneOffset;
2525
import java.util.List;
26+
import java.util.Map;
27+
import java.util.stream.Stream;
2628
import org.jooq.Configuration;
2729
import org.jooq.Record;
2830
import org.sonarsource.sonarlint.core.commons.KnownFinding;
2931
import org.sonarsource.sonarlint.core.commons.KnownFindingType;
3032
import org.sonarsource.sonarlint.core.commons.LineWithHash;
3133
import org.sonarsource.sonarlint.core.commons.api.TextRangeWithHash;
3234
import org.sonarsource.sonarlint.core.commons.storage.SonarLintDatabase;
35+
import org.sonarsource.sonarlint.core.commons.storage.model.tables.records.KnownFindingsRecord;
3336

3437
import static org.sonarsource.sonarlint.core.commons.storage.model.Tables.KNOWN_FINDINGS;
3538

@@ -41,6 +44,47 @@ public KnownFindingsRepository(SonarLintDatabase database) {
4144
this.database = database;
4245
}
4346

47+
public void storeFindings(Map<String, Map<Path, Findings>> findingsPerFilePerConfigScopeId) {
48+
database.dsl().batchMerge(findingsPerFilePerConfigScopeId.entrySet().stream()
49+
.flatMap(configScopeEntry -> {
50+
var configScopeId = configScopeEntry.getKey();
51+
return configScopeEntry.getValue().entrySet().stream()
52+
.flatMap(fileEntry -> {
53+
var filePath = fileEntry.getKey();
54+
var findings = fileEntry.getValue();
55+
return Stream.<KnownFindingsRecord>concat(
56+
findings.issues().stream()
57+
.map(finding -> createRecord(finding, configScopeId, filePath, KnownFindingType.ISSUE)),
58+
findings.hotspots().stream()
59+
.map(finding -> createRecord(finding, configScopeId, filePath, KnownFindingType.HOTSPOT)));
60+
});
61+
})
62+
.toList())
63+
.execute();
64+
}
65+
66+
private static KnownFindingsRecord createRecord(KnownFinding finding, String configScopeId, Path filePath, KnownFindingType type) {
67+
var textRangeWithHash = finding.getTextRangeWithHash();
68+
var lineWithHash = finding.getLineWithHash();
69+
var introductionDate = LocalDateTime.ofInstant(finding.getIntroductionDate(), ZoneOffset.UTC);
70+
return new KnownFindingsRecord(
71+
finding.getId(),
72+
configScopeId,
73+
filePath.toString(),
74+
finding.getServerKey(),
75+
finding.getRuleKey(),
76+
finding.getMessage(),
77+
introductionDate,
78+
type.name(),
79+
textRangeWithHash == null ? null : textRangeWithHash.getStartLine(),
80+
textRangeWithHash == null ? null : textRangeWithHash.getStartLineOffset(),
81+
textRangeWithHash == null ? null : textRangeWithHash.getEndLine(),
82+
textRangeWithHash == null ? null : textRangeWithHash.getEndLineOffset(),
83+
textRangeWithHash == null ? null : textRangeWithHash.getHash(),
84+
lineWithHash == null ? null : lineWithHash.getNumber(),
85+
lineWithHash == null ? null : lineWithHash.getHash());
86+
}
87+
4488
public void storeKnownIssues(String configurationScopeId, Path clientRelativePath, List<KnownFinding> newKnownIssues) {
4589
storeKnownFindings(configurationScopeId, clientRelativePath, newKnownIssues, KnownFindingType.ISSUE);
4690
}
@@ -69,7 +113,7 @@ private void storeKnownFindings(String configurationScopeId, Path clientRelative
69113
var lineWithHash = finding.getLineWithHash();
70114
var line = lineWithHash == null ? null : lineWithHash.getNumber();
71115
var lineHash = lineWithHash == null ? null : lineWithHash.getHash();
72-
var introDate = LocalDateTime.ofInstant(finding.getIntroductionDate(), ZoneId.systemDefault());
116+
var introDate = LocalDateTime.ofInstant(finding.getIntroductionDate(), ZoneOffset.UTC);
73117
trx.dsl().mergeInto(KNOWN_FINDINGS)
74118
.using(trx.dsl().selectOne())
75119
.on(KNOWN_FINDINGS.ID.eq(finding.getId()))
@@ -95,8 +139,7 @@ private void storeKnownFindings(String configurationScopeId, Path clientRelative
95139
.values(finding.getId(), configurationScopeId, clientRelativePath.toString(), finding.getServerKey(), finding.getRuleKey(),
96140
finding.getMessage(), introDate, type.name(),
97141
startLine, startLineOffset, endLine, endLineOffset, textRangeHash,
98-
line, lineHash
99-
)
142+
line, lineHash)
100143
.execute();
101144
}));
102145
}
@@ -106,8 +149,7 @@ private List<KnownFinding> getKnownFindingsForFile(String configurationScopeId,
106149
.selectFrom(KNOWN_FINDINGS)
107150
.where(KNOWN_FINDINGS.CONFIGURATION_SCOPE_ID.eq(configurationScopeId)
108151
.and(KNOWN_FINDINGS.IDE_RELATIVE_FILE_PATH.eq(filePath.toString()))
109-
.and(KNOWN_FINDINGS.FINDING_TYPE.eq(type.name()))
110-
)
152+
.and(KNOWN_FINDINGS.FINDING_TYPE.eq(type.name())))
111153
.fetch();
112154
return issuesInFile.stream()
113155
.map(KnownFindingsRepository::recordToKnownFinding)
@@ -116,7 +158,7 @@ private List<KnownFinding> getKnownFindingsForFile(String configurationScopeId,
116158

117159
private static KnownFinding recordToKnownFinding(Record rec) {
118160
var id = rec.get(KNOWN_FINDINGS.ID);
119-
var introductionDate = rec.get(KNOWN_FINDINGS.INTRODUCTION_DATE).atZone(ZoneId.systemDefault()).toInstant();
161+
var introductionDate = rec.get(KNOWN_FINDINGS.INTRODUCTION_DATE).toInstant(ZoneOffset.UTC);
120162
var textRangeWithHash = getTextRangeWithHash(rec);
121163
var lineWithHash = getLineWithHash(rec);
122164
return new KnownFinding(
@@ -125,25 +167,27 @@ private static KnownFinding recordToKnownFinding(Record rec) {
125167
textRangeWithHash, lineWithHash,
126168
rec.get(KNOWN_FINDINGS.RULE_KEY),
127169
rec.get(KNOWN_FINDINGS.MESSAGE),
128-
introductionDate
129-
);
170+
introductionDate);
130171
}
131172

132173
private static LineWithHash getLineWithHash(Record rec) {
133-
if (rec.get(KNOWN_FINDINGS.LINE) == null) return null;
134174
var line = rec.get(KNOWN_FINDINGS.LINE);
175+
if (line == null) {
176+
return null;
177+
}
135178
var hash = rec.get(KNOWN_FINDINGS.LINE_HASH);
136179
return new LineWithHash(line, hash);
137180
}
138181

139182
private static TextRangeWithHash getTextRangeWithHash(Record rec) {
140-
if (rec.get(KNOWN_FINDINGS.START_LINE) == null) return null;
141183
var startLine = rec.get(KNOWN_FINDINGS.START_LINE);
184+
if (startLine == null) {
185+
return null;
186+
}
142187
var endLine = rec.get(KNOWN_FINDINGS.END_LINE);
143188
var startLineOffset = rec.get(KNOWN_FINDINGS.START_LINE_OFFSET);
144189
var endLineOffset = rec.get(KNOWN_FINDINGS.END_LINE_OFFSET);
145190
var hash = rec.get(KNOWN_FINDINGS.TEXT_RANGE_HASH);
146191
return new TextRangeWithHash(startLine, startLineOffset, endLine, endLineOffset, hash);
147192
}
148-
149193
}

‎backend/core/src/main/java/org/sonarsource/sonarlint/core/tracking/KnownFindingsStorageService.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.sonarsource.sonarlint.core.tracking;
2121

2222
import java.io.IOException;
23+
import java.nio.file.Files;
2324
import java.nio.file.Path;
2425
import java.util.concurrent.atomic.AtomicReference;
2526
import javax.annotation.PreDestroy;
@@ -36,6 +37,10 @@ public KnownFindingsStorageService(UserPaths userPaths) {
3637
this.workDir = userPaths.getWorkDir();
3738
}
3839

40+
public boolean exists() {
41+
return Files.exists(projectsStorageBaseDir.resolve(XodusKnownFindingsStore.BACKUP_TAR_GZ));
42+
}
43+
3944
public synchronized XodusKnownFindingsStore get() {
4045
var store = trackedIssuesStore.get();
4146
if (store == null) {
@@ -58,4 +63,10 @@ public void close() {
5863
}
5964
}
6065

66+
public void delete() {
67+
var store = trackedIssuesStore.getAndSet(null);
68+
if (store != null) {
69+
store.delete();
70+
}
71+
}
6172
}

‎backend/core/src/main/java/org/sonarsource/sonarlint/core/tracking/TrackingService.java‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
*/
2020
package org.sonarsource.sonarlint.core.tracking;
2121

22+
import jakarta.annotation.PostConstruct;
2223
import java.net.URI;
2324
import java.nio.file.Path;
2425
import java.time.Instant;
@@ -117,6 +118,17 @@ public TrackingService(SonarLintRpcClient client, ConfigurationRepository config
117118
this.databaseService = databaseService;
118119
}
119120

121+
@PostConstruct
122+
public void migrateData() {
123+
if (dogfoodEnvironmentDetectionService.isDogfoodEnvironment() && knownFindingsStorageService.exists()) {
124+
var repository = new KnownFindingsRepository(databaseService.getDatabase());
125+
var xodusKnownFindingsStore = knownFindingsStorageService.get();
126+
var findingsPerConfigScope = xodusKnownFindingsStore.loadAll();
127+
repository.storeFindings(findingsPerConfigScope);
128+
knownFindingsStorageService.delete();
129+
}
130+
}
131+
120132
@EventListener
121133
public void onAnalysisStarted(AnalysisStartedEvent event) {
122134
var configurationScopeId = event.getConfigurationScopeId();
@@ -168,7 +180,6 @@ private MatchingResult matchWithServerFindings(String configurationScopeId, Matc
168180
var effectiveBindingOpt = configurationRepository.getEffectiveBinding(configurationScopeId);
169181
var activeBranchOpt = branchTrackingService.awaitEffectiveSonarProjectBranch(configurationScopeId);
170182
var translationOpt = pathTranslationService.getOrComputePathTranslation(configurationScopeId);
171-
var knownFindingsStore = knownFindingsStorageService.get();
172183
var issuesToReport = matchingSession.getIssuesPerFile();
173184
var hotspotsToReport = matchingSession.getSecurityHotspotsPerFile();
174185
if (effectiveBindingOpt.isPresent() && activeBranchOpt.isPresent() && translationOpt.isPresent()) {
@@ -191,34 +202,34 @@ private MatchingResult matchWithServerFindings(String configurationScopeId, Matc
191202
return Map.entry(ideRelativePath, matches);
192203
}).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
193204
}
194-
issuesToReport.forEach((clientRelativePath, trackedIssues) -> storeTrackedIssues(knownFindingsStore, configurationScopeId, clientRelativePath, trackedIssues));
195-
hotspotsToReport.forEach((clientRelativePath, trackedHotspots) -> storeTrackedSecurityHotspots(knownFindingsStore, configurationScopeId, clientRelativePath, trackedHotspots));
205+
issuesToReport.forEach((clientRelativePath, trackedIssues) -> storeTrackedIssues(configurationScopeId, clientRelativePath, trackedIssues));
206+
hotspotsToReport.forEach((clientRelativePath, trackedHotspots) -> storeTrackedSecurityHotspots(configurationScopeId, clientRelativePath, trackedHotspots));
196207
eventPublisher.publishEvent(new MatchingSessionEndedEvent(matchingSession.countNewIssues(), matchingSession.countRemainingUnmatchedIssues()));
197208
return new MatchingResult(issuesToReport, hotspotsToReport);
198209
}
199210

200-
private void storeTrackedIssues(XodusKnownFindingsStore knownIssuesStore, String configurationScopeId, Path clientRelativePath, Collection<TrackedIssue> newKnownIssues) {
211+
private void storeTrackedIssues(String configurationScopeId, Path clientRelativePath, Collection<TrackedIssue> newKnownIssues) {
201212
if (dogfoodEnvironmentDetectionService.isDogfoodEnvironment()) {
202213
var knownFindingsRepository = new KnownFindingsRepository(databaseService.getDatabase());
203214
knownFindingsRepository.storeKnownIssues(configurationScopeId, clientRelativePath,
204215
newKnownIssues.stream().map(i -> new KnownFinding(i.getId(), i.getServerKey(), i.getTextRangeWithHash(), i.getLineWithHash(), i.getRuleKey(), i.getMessage(),
205216
i.getIntroductionDate())).toList());
206217
} else {
207-
knownIssuesStore.storeKnownIssues(configurationScopeId, clientRelativePath,
218+
knownFindingsStorageService.get().storeKnownIssues(configurationScopeId, clientRelativePath,
208219
newKnownIssues.stream().map(i -> new KnownFinding(i.getId(), i.getServerKey(), i.getTextRangeWithHash(), i.getLineWithHash(), i.getRuleKey(), i.getMessage(),
209220
i.getIntroductionDate())).toList());
210221
}
211222
}
212223

213-
private void storeTrackedSecurityHotspots(XodusKnownFindingsStore knownIssuesStore, String configurationScopeId, Path clientRelativePath,
224+
private void storeTrackedSecurityHotspots(String configurationScopeId, Path clientRelativePath,
214225
Collection<TrackedIssue> newKnownSecurityHotspots) {
215226
if (dogfoodEnvironmentDetectionService.isDogfoodEnvironment()) {
216227
var knownFindingsRepository = new KnownFindingsRepository(databaseService.getDatabase());
217228
knownFindingsRepository.storeKnownSecurityHotspots(configurationScopeId, clientRelativePath,
218229
newKnownSecurityHotspots.stream().map(i -> new KnownFinding(i.getId(), i.getServerKey(), i.getTextRangeWithHash(), i.getLineWithHash(), i.getRuleKey(), i.getMessage(),
219230
i.getIntroductionDate())).toList());
220231
} else {
221-
knownIssuesStore.storeKnownSecurityHotspots(configurationScopeId, clientRelativePath,
232+
knownFindingsStorageService.get().storeKnownSecurityHotspots(configurationScopeId, clientRelativePath,
222233
newKnownSecurityHotspots.stream().map(i -> new KnownFinding(i.getId(), i.getServerKey(), i.getTextRangeWithHash(), i.getLineWithHash(), i.getRuleKey(), i.getMessage(),
223234
i.getIntroductionDate())).toList());
224235
}
@@ -301,7 +312,6 @@ private static TrackedIssue updateTrackedIssueWithLocalOnlyIssueData(TrackedIssu
301312
}
302313

303314
private MatchingSession startMatchingSession(String configurationScopeId, Set<Path> fileRelativePaths, Set<URI> fileUris, UnaryOperator<String> fileContentProvider) {
304-
var knownFindingsStore = knownFindingsStorageService.get();
305315
var dogfoodEnvironment = dogfoodEnvironmentDetectionService.isDogfoodEnvironment();
306316
Map<Path, List<KnownFinding>> issuesByRelativePath;
307317
Map<Path, List<KnownFinding>> hotspotsByRelativePath;
@@ -312,6 +322,7 @@ private MatchingSession startMatchingSession(String configurationScopeId, Set<Pa
312322
hotspotsByRelativePath = fileRelativePaths.stream()
313323
.collect(toMap(Function.identity(), relativePath -> knownFindingsRepository.loadSecurityHotspotsForFile(configurationScopeId, relativePath)));
314324
} else {
325+
var knownFindingsStore = knownFindingsStorageService.get();
315326
issuesByRelativePath = fileRelativePaths.stream()
316327
.collect(toMap(Function.identity(), relativePath -> knownFindingsStore.loadIssuesForFile(configurationScopeId, relativePath)));
317328
hotspotsByRelativePath = fileRelativePaths.stream()

‎backend/core/src/main/java/org/sonarsource/sonarlint/core/tracking/XodusKnownFindingsStore.java‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,12 @@
2222
import java.io.IOException;
2323
import java.nio.file.Files;
2424
import java.nio.file.Path;
25+
import java.nio.file.Paths;
2526
import java.nio.file.StandardCopyOption;
2627
import java.time.Instant;
2728
import java.util.Collections;
2829
import java.util.List;
30+
import java.util.Map;
2931
import java.util.Optional;
3032
import java.util.UUID;
3133
import java.util.stream.StreamSupport;
@@ -42,11 +44,16 @@
4244
import org.sonarsource.sonarlint.core.commons.LineWithHash;
4345
import org.sonarsource.sonarlint.core.commons.api.TextRangeWithHash;
4446
import org.sonarsource.sonarlint.core.commons.log.SonarLintLogger;
47+
import org.sonarsource.sonarlint.core.commons.storage.repository.Findings;
4548
import org.sonarsource.sonarlint.core.serverconnection.storage.InstantBinding;
4649
import org.sonarsource.sonarlint.core.serverconnection.storage.TarGzUtils;
4750
import org.sonarsource.sonarlint.core.serverconnection.storage.UuidBinding;
4851

4952
import static java.util.Objects.requireNonNull;
53+
import static java.util.stream.Collectors.flatMapping;
54+
import static java.util.stream.Collectors.groupingBy;
55+
import static java.util.stream.Collectors.toMap;
56+
import static org.sonarsource.sonarlint.core.commons.storage.XodusPurgeUtils.deleteInFolderWithPattern;
5057
import static org.sonarsource.sonarlint.core.commons.storage.XodusPurgeUtils.purgeOldTemporaryFiles;
5158

5259
public class XodusKnownFindingsStore {
@@ -73,13 +80,15 @@ public class XodusKnownFindingsStore {
7380
private static final String END_LINE_PROPERTY_NAME = "endLine";
7481
private static final String END_LINE_OFFSET_PROPERTY_NAME = "endLineOffset";
7582
private static final String MESSAGE_BLOB_NAME = "message";
76-
private static final String BACKUP_TAR_GZ = "known_findings_backup.tar.gz";
83+
static final String BACKUP_TAR_GZ = "known_findings_backup.tar.gz";
7784
private final PersistentEntityStore entityStore;
7885
private final Path backupFile;
7986
private final Path xodusDbDir;
8087
private static final SonarLintLogger LOG = SonarLintLogger.get();
88+
private final Path workDir;
8189

8290
public XodusKnownFindingsStore(Path backupDir, Path workDir) throws IOException {
91+
this.workDir = workDir;
8392
xodusDbDir = Files.createTempDirectory(workDir, KNOWN_FINDINGS_STORE);
8493
purgeOldTemporaryFiles(workDir, PURGE_NUMBER_OF_DAYS, KNOWN_FINDINGS_STORE + "*");
8594
backupFile = backupDir.resolve(BACKUP_TAR_GZ);
@@ -99,6 +108,21 @@ public XodusKnownFindingsStore(Path backupDir, Path workDir) throws IOException
99108
});
100109
}
101110

111+
public Map<String, Map<Path, Findings>> loadAll() {
112+
return entityStore.computeInReadonlyTransaction(txn -> StreamSupport.stream(txn.getAll(CONFIGURATION_SCOPE_ID_ENTITY_TYPE).spliterator(), false)
113+
.collect(groupingBy(
114+
e -> (String) requireNonNull(e.getProperty(NAME_PROPERTY_NAME)),
115+
flatMapping(e -> StreamSupport.stream(e.getLinks(CONFIGURATION_SCOPE_ID_TO_FILES_LINK_NAME).spliterator(), false),
116+
toMap(
117+
f -> Paths.get((String) requireNonNull(f.getProperty(PATH_PROPERTY_NAME))),
118+
f -> new Findings(
119+
StreamSupport.stream(f.getLinks(FILE_TO_ISSUES_LINK_NAME).spliterator(), false)
120+
.map(XodusKnownFindingsStore::adapt).toList(),
121+
StreamSupport.stream(f.getLinks(FILE_TO_SECURITY_HOTSPOTS_LINK_NAME).spliterator(), false)
122+
.map(XodusKnownFindingsStore::adapt).toList()),
123+
Findings::mergeWith)))));
124+
}
125+
102126
public List<KnownFinding> loadIssuesForFile(String configurationScopeId, Path filePath) {
103127
return loadFindingsForFile(configurationScopeId, filePath, FILE_TO_ISSUES_LINK_NAME);
104128
}
@@ -229,7 +253,7 @@ private static void updateOrCreateFinding(Entity fileEntity, KnownFinding findin
229253
}
230254
}
231255

232-
public void backup() {
256+
private void backup() {
233257
LOG.debug("Creating backup of known findings database in {}", backupFile);
234258
try {
235259
var backupTmp = CompressBackupUtil.backup(entityStore, backupFile.getParent().toFile(), "known_findings_backup", false);
@@ -262,4 +286,12 @@ private void storeKnownFindings(String configurationScopeId, Path clientRelative
262286
newKnownFindings.forEach(finding -> updateOrCreateFinding(fileEntity, finding, fileToFindingsLinkName, txn));
263287
});
264288
}
289+
290+
public void delete() {
291+
entityStore.close();
292+
FileUtils.deleteQuietly(xodusDbDir.toFile());
293+
FileUtils.deleteQuietly(backupFile.toFile());
294+
deleteInFolderWithPattern(workDir, KNOWN_FINDINGS_STORE + "*");
295+
deleteInFolderWithPattern(backupFile.getParent(), "known_findings_backup*");
296+
}
265297
}

0 commit comments

Comments
 (0)