Skip to content

Commit d8ccc6e

Browse files
damien-urruty-sonarsourcedamien-urruty-sonarsource
committed
SLCORE-1819 Migrate existing local-only findings to the new database
1 parent f029003 commit d8ccc6e

File tree

8 files changed

+152
-12
lines changed

8 files changed

+152
-12
lines changed

‎backend/commons/src/main/java/org/sonarsource/sonarlint/core/commons/storage/XodusPurgeUtils.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,16 @@ public static void purgeOldTemporaryFiles(Path workDir, Integer purgeDays, Strin
5050
}
5151
}
5252

53+
public static void deleteInFolderWithPattern(Path folder, String pattern) {
54+
if (Files.exists(folder)) {
55+
try (var stream = Files.newDirectoryStream(folder, pattern)) {
56+
for (var path : stream) {
57+
FileUtils.deleteQuietly(path.toFile());
58+
}
59+
} catch (Exception e) {
60+
LOG.error("Unable to remove files in {} for pattern {}", folder, pattern);
61+
}
62+
}
63+
}
64+
5365
}

‎backend/commons/src/main/java/org/sonarsource/sonarlint/core/commons/storage/repository/LocalOnlyIssuesRepository.java‎

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@
2222
import java.nio.file.Path;
2323
import java.time.Instant;
2424
import java.time.LocalDateTime;
25-
import java.time.ZoneId;
25+
import java.time.ZoneOffset;
2626
import java.util.List;
27+
import java.util.Map;
2728
import java.util.Optional;
2829
import java.util.UUID;
2930
import org.jooq.Configuration;
@@ -34,6 +35,7 @@
3435
import org.sonarsource.sonarlint.core.commons.LocalOnlyIssueResolution;
3536
import org.sonarsource.sonarlint.core.commons.api.TextRangeWithHash;
3637
import org.sonarsource.sonarlint.core.commons.storage.SonarLintDatabase;
38+
import org.sonarsource.sonarlint.core.commons.storage.model.tables.records.LocalOnlyIssuesRecord;
3739

3840
import static org.sonarsource.sonarlint.core.commons.storage.model.Tables.LOCAL_ONLY_ISSUES;
3941

@@ -66,6 +68,38 @@ public List<LocalOnlyIssue> loadAll(String configurationScopeId) {
6668
.toList();
6769
}
6870

71+
public void storeIssues(Map<String, List<LocalOnlyIssue>> issuesPerConfigScopeId) {
72+
database.dsl().batchMerge(issuesPerConfigScopeId.entrySet().stream()
73+
.flatMap(entry -> {
74+
var configScopeId = entry.getKey();
75+
return entry.getValue().stream().map(
76+
issue -> {
77+
var resolution = issue.getResolution();
78+
var textRangeWithHash = issue.getTextRangeWithHash();
79+
var lineWithHash = issue.getLineWithHash();
80+
return new LocalOnlyIssuesRecord(
81+
issue.getId(),
82+
configScopeId,
83+
issue.getServerRelativePath().toString(),
84+
issue.getRuleKey(),
85+
issue.getMessage(),
86+
resolution == null ? null : resolution.getStatus().name(),
87+
resolution == null ? null : LocalDateTime.ofInstant(resolution.getResolutionDate(), ZoneOffset.UTC),
88+
resolution == null ? null : resolution.getComment(),
89+
textRangeWithHash == null ? null : textRangeWithHash.getStartLine(),
90+
textRangeWithHash == null ? null : textRangeWithHash.getStartLineOffset(),
91+
textRangeWithHash == null ? null : textRangeWithHash.getEndLine(),
92+
textRangeWithHash == null ? null : textRangeWithHash.getEndLineOffset(),
93+
textRangeWithHash == null ? null : textRangeWithHash.getHash(),
94+
lineWithHash == null ? null : lineWithHash.getNumber(),
95+
lineWithHash == null ? null : lineWithHash.getHash());
96+
97+
});
98+
})
99+
.toList())
100+
.execute();
101+
}
102+
69103
public void storeLocalOnlyIssue(String configurationScopeId, LocalOnlyIssue issue) {
70104
database.dsl().transaction((Configuration trx) -> {
71105
var textRangeWithHash = issue.getTextRangeWithHash();
@@ -81,7 +115,7 @@ public void storeLocalOnlyIssue(String configurationScopeId, LocalOnlyIssue issu
81115

82116
var resolution = issue.getResolution();
83117
var resolutionStatus = resolution == null ? null : resolution.getStatus().name();
84-
var resolutionDate = resolution == null ? null : LocalDateTime.ofInstant(resolution.getResolutionDate(), ZoneId.systemDefault());
118+
var resolutionDate = resolution == null ? null : LocalDateTime.ofInstant(resolution.getResolutionDate(), ZoneOffset.UTC);
85119
var comment = resolution == null ? null : resolution.getComment();
86120

87121
trx.dsl().mergeInto(LOCAL_ONLY_ISSUES)
@@ -164,7 +198,7 @@ public Optional<LocalOnlyIssue> find(UUID issueId) {
164198
}
165199

166200
public void purgeIssuesOlderThan(Instant limit) {
167-
var limitDateTime = LocalDateTime.ofInstant(limit, ZoneId.systemDefault());
201+
var limitDateTime = LocalDateTime.ofInstant(limit, ZoneOffset.UTC);
168202
database.dsl()
169203
.deleteFrom(LOCAL_ONLY_ISSUES)
170204
.where(LOCAL_ONLY_ISSUES.RESOLUTION_DATE.isNotNull()
@@ -186,7 +220,7 @@ private static LocalOnlyIssue recordToLocalOnlyIssue(Record rec) {
186220
var resolutionDate = rec.get(LOCAL_ONLY_ISSUES.RESOLUTION_DATE);
187221
if (resolutionStatus != null && resolutionDate != null) {
188222
var status = IssueStatus.valueOf(resolutionStatus);
189-
var instant = resolutionDate.atZone(ZoneId.systemDefault()).toInstant();
223+
var instant = resolutionDate.toInstant(ZoneOffset.UTC);
190224
var comment = rec.get(LOCAL_ONLY_ISSUES.COMMENT);
191225
resolution = new LocalOnlyIssueResolution(status, instant, comment);
192226
}

‎backend/commons/src/test/java/org/sonarsource/sonarlint/core/commons/storage/repository/LocalOnlyIssuesRepositoryTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ void should_return_empty_when_issue_not_found() {
169169
}
170170

171171
@Test
172-
void should_remove_issue(@TempDir Path temp) {
172+
void should_remove_issue() {
173173
var configScopeId = "configScopeId";
174174
var filePath = Path.of("/file/path");
175175
var issueUuid1 = UUID.randomUUID();

‎backend/core/src/main/java/org/sonarsource/sonarlint/core/issue/IssueService.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
*/
2020
package org.sonarsource.sonarlint.core.issue;
2121

22+
import jakarta.annotation.PostConstruct;
2223
import java.nio.file.Path;
2324
import java.util.ArrayList;
2425
import java.util.Collections;
@@ -135,6 +136,17 @@ public IssueService(ConfigurationRepository configurationRepository, SonarQubeCl
135136
this.databaseService = databaseService;
136137
}
137138

139+
@PostConstruct
140+
public void migrateData() {
141+
if (dogfoodEnvironmentDetectionService.isDogfoodEnvironment() && localOnlyIssueStorageService.exists()) {
142+
var repository = new LocalOnlyIssuesRepository(databaseService.getDatabase());
143+
var xodusLocalOnlyIssueStore = localOnlyIssueStorageService.get();
144+
var issuesPerConfigScope = xodusLocalOnlyIssueStore.loadAll();
145+
repository.storeIssues(issuesPerConfigScope);
146+
localOnlyIssueStorageService.delete();
147+
}
148+
}
149+
138150
public void changeStatus(String configurationScopeId, String issueKey, ResolutionStatus newStatus, boolean isTaintIssue, SonarLintCancelMonitor cancelMonitor) {
139151
var binding = configurationRepository.getEffectiveBindingOrThrow(configurationScopeId);
140152
var serverConnection = sonarQubeClientManager.getClientOrThrow(binding.connectionId());
@@ -303,15 +315,15 @@ private void removeAllIssuesForFile(String configurationScopeId, Path filePath,
303315
var issuesToSync = subtract(allIssues, issuesForFile);
304316
var binding = configurationRepository.getEffectiveBindingOrThrow(configurationScopeId);
305317
sonarQubeClientManager.getClientOrThrow(binding.connectionId())
306-
.withClientApi(serverApi -> serverApi.issue().anticipatedTransitions(binding.sonarProjectKey(), issuesToSync, cancelMonitor));
318+
.withClientApi(serverApi -> serverApi.issue().anticipatedTransitions(binding.sonarProjectKey(), issuesToSync, cancelMonitor));
307319
}
308320

309321
private void removeIssueOnServer(String configurationScopeId, UUID issueId, SonarLintCancelMonitor cancelMonitor) {
310322
var allIssues = loadAllLocalOnlyIssues(configurationScopeId);
311323
var issuesToSync = allIssues.stream().filter(it -> !it.getId().equals(issueId)).toList();
312324
var binding = configurationRepository.getEffectiveBindingOrThrow(configurationScopeId);
313325
sonarQubeClientManager.getClientOrThrow(binding.connectionId())
314-
.withClientApi(serverApi -> serverApi.issue().anticipatedTransitions(binding.sonarProjectKey(), issuesToSync, cancelMonitor));
326+
.withClientApi(serverApi -> serverApi.issue().anticipatedTransitions(binding.sonarProjectKey(), issuesToSync, cancelMonitor));
315327
}
316328

317329
private void setCommentOnLocalOnlyIssue(String configurationScopeId, UUID issueId, String comment, SonarLintCancelMonitor cancelMonitor) {
@@ -325,7 +337,7 @@ private void setCommentOnLocalOnlyIssue(String configurationScopeId, UUID issueI
325337
issuesToSync.replaceAll(issue -> issue.getId().equals(issueId) ? commentedIssue : issue);
326338
var binding = configurationRepository.getEffectiveBindingOrThrow(configurationScopeId);
327339
sonarQubeClientManager.getClientOrThrow(binding.connectionId())
328-
.withClientApi(serverApi -> serverApi.issue().anticipatedTransitions(binding.sonarProjectKey(), issuesToSync, cancelMonitor));
340+
.withClientApi(serverApi -> serverApi.issue().anticipatedTransitions(binding.sonarProjectKey(), issuesToSync, cancelMonitor));
329341
storeLocalOnlyIssue(configurationScopeId, commentedIssue);
330342
}
331343
} else {

‎backend/core/src/main/java/org/sonarsource/sonarlint/core/local/only/LocalOnlyIssueStorageService.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import jakarta.annotation.PostConstruct;
2323
import jakarta.annotation.PreDestroy;
2424
import java.io.IOException;
25+
import java.nio.file.Files;
2526
import java.nio.file.Path;
2627
import java.time.Instant;
2728
import java.time.temporal.ChronoUnit;
@@ -40,7 +41,13 @@ public LocalOnlyIssueStorageService(UserPaths userPaths) {
4041

4142
@PostConstruct
4243
public void purgeOldIssues() {
43-
get().purgeIssuesOlderThan(Instant.now().minus(7, ChronoUnit.DAYS));
44+
if (exists()) {
45+
get().purgeIssuesOlderThan(Instant.now().minus(7, ChronoUnit.DAYS));
46+
}
47+
}
48+
49+
public boolean exists() {
50+
return Files.exists(projectsStorageBaseDir.resolve(XodusLocalOnlyIssueStore.BACKUP_TAR_GZ));
4451
}
4552

4653
public XodusLocalOnlyIssueStore get() {
@@ -62,4 +69,10 @@ public void close() {
6269
}
6370
}
6471

72+
public void delete() {
73+
if (localOnlyIssueStore != null) {
74+
localOnlyIssueStore.delete();
75+
localOnlyIssueStore = null;
76+
}
77+
}
6578
}

‎backend/core/src/main/java/org/sonarsource/sonarlint/core/local/only/XodusLocalOnlyIssueStore.java‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import java.util.ArrayList;
2828
import java.util.Collections;
2929
import java.util.List;
30+
import java.util.Map;
3031
import java.util.Optional;
3132
import java.util.UUID;
3233
import java.util.stream.StreamSupport;
@@ -50,6 +51,10 @@
5051
import org.sonarsource.sonarlint.core.serverconnection.storage.UuidBinding;
5152

5253
import static java.util.Objects.requireNonNull;
54+
import static java.util.stream.Collectors.flatMapping;
55+
import static java.util.stream.Collectors.groupingBy;
56+
import static java.util.stream.Collectors.toList;
57+
import static org.sonarsource.sonarlint.core.commons.storage.XodusPurgeUtils.deleteInFolderWithPattern;
5358
import static org.sonarsource.sonarlint.core.commons.storage.XodusPurgeUtils.purgeOldTemporaryFiles;
5459

5560
public class XodusLocalOnlyIssueStore {
@@ -76,13 +81,15 @@ public class XodusLocalOnlyIssueStore {
7681
private static final String END_LINE_PROPERTY_NAME = "endLine";
7782
private static final String END_LINE_OFFSET_PROPERTY_NAME = "endLineOffset";
7883
private static final String MESSAGE_BLOB_NAME = "message";
79-
private static final String BACKUP_TAR_GZ = "local_only_issue_backup.tar.gz";
84+
static final String BACKUP_TAR_GZ = "local_only_issue_backup.tar.gz";
8085
private final PersistentEntityStore entityStore;
8186
private final Path backupFile;
8287
private final Path xodusDbDir;
8388
private static final SonarLintLogger LOG = SonarLintLogger.get();
89+
private final Path workDir;
8490

8591
public XodusLocalOnlyIssueStore(Path backupDir, Path workDir) throws IOException {
92+
this.workDir = workDir;
8693
xodusDbDir = Files.createTempDirectory(workDir, LOCAL_ONLY_ISSUE);
8794
purgeOldTemporaryFiles(workDir, PURGE_NUMBER_OF_DAYS, LOCAL_ONLY_ISSUE + "*");
8895
backupFile = backupDir.resolve(BACKUP_TAR_GZ);
@@ -114,6 +121,16 @@ public List<LocalOnlyIssue> loadForFile(String configurationScopeId, Path filePa
114121
.orElseGet(Collections::emptyList));
115122
}
116123

124+
public Map<String, List<LocalOnlyIssue>> loadAll() {
125+
return entityStore.computeInReadonlyTransaction(txn -> StreamSupport.stream(txn.getAll(CONFIGURATION_SCOPE_ID_ENTITY_TYPE).spliterator(), false)
126+
.collect(groupingBy(
127+
e -> (String) requireNonNull(e.getProperty(NAME_PROPERTY_NAME)),
128+
flatMapping(e -> StreamSupport.stream(e.getLinks(CONFIGURATION_SCOPE_ID_TO_FILES_LINK_NAME).spliterator(), false)
129+
.flatMap(file -> StreamSupport.stream(file.getLinks(XodusLocalOnlyIssueStore.FILE_TO_ISSUES_LINK_NAME).spliterator(), false)
130+
.map(XodusLocalOnlyIssueStore::adapt)),
131+
toList()))));
132+
}
133+
117134
public List<LocalOnlyIssue> loadAll(String configurationScopeId) {
118135
return entityStore.computeInReadonlyTransaction(txn -> findUnique(txn, CONFIGURATION_SCOPE_ID_ENTITY_TYPE, NAME_PROPERTY_NAME, configurationScopeId)
119136
.map(configScopeId -> configScopeId.getLinks(CONFIGURATION_SCOPE_ID_TO_FILES_LINK_NAME))
@@ -321,4 +338,12 @@ public void close() {
321338
entityStore.close();
322339
FileUtils.deleteQuietly(xodusDbDir.toFile());
323340
}
341+
342+
public void delete() {
343+
entityStore.close();
344+
FileUtils.deleteQuietly(xodusDbDir.toFile());
345+
FileUtils.deleteQuietly(backupFile.toFile());
346+
deleteInFolderWithPattern(workDir, LOCAL_ONLY_ISSUE + "*");
347+
deleteInFolderWithPattern(backupFile.getParent(), "local_only_issue_backup*");
348+
}
324349
}

‎medium-tests/src/test/java/mediumtest/issues/LocalOnlyResolvedIssuesStorageMediumTests.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,36 @@
1919
*/
2020
package mediumtest.issues;
2121

22+
import java.nio.file.Paths;
2223
import java.time.Instant;
2324
import java.time.temporal.ChronoUnit;
25+
import org.junit.jupiter.api.BeforeEach;
26+
import org.junit.jupiter.api.extension.ExtendWith;
2427
import org.sonarsource.sonarlint.core.commons.RuleType;
2528
import org.sonarsource.sonarlint.core.commons.api.TextRangeWithHash;
29+
import org.sonarsource.sonarlint.core.commons.storage.repository.LocalOnlyIssuesRepository;
2630
import org.sonarsource.sonarlint.core.test.utils.junit5.SonarLintTest;
2731
import org.sonarsource.sonarlint.core.test.utils.junit5.SonarLintTestHarness;
32+
import uk.org.webcompere.systemstubs.environment.EnvironmentVariables;
33+
import uk.org.webcompere.systemstubs.jupiter.SystemStub;
34+
import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension;
2835

2936
import static mediumtest.fixtures.LocalOnlyIssueFixtures.aLocalOnlyIssueResolved;
3037
import static org.assertj.core.api.Assertions.assertThat;
38+
import static org.sonarsource.sonarlint.core.commons.monitoring.DogfoodEnvironmentDetectionService.SONARSOURCE_DOGFOODING_ENV_VAR_KEY;
3139
import static org.sonarsource.sonarlint.core.test.utils.storage.ServerIssueFixtures.aServerIssue;
3240

41+
@ExtendWith(SystemStubsExtension.class)
3342
class LocalOnlyResolvedIssuesStorageMediumTests {
3443

44+
@SystemStub
45+
EnvironmentVariables environmentVariables;
46+
47+
@BeforeEach
48+
void prepare() {
49+
environmentVariables.remove(SONARSOURCE_DOGFOODING_ENV_VAR_KEY);
50+
}
51+
3552
@SonarLintTest
3653
void it_should_purge_local_only_stored_issues_resolved_more_than_one_week_ago_at_startup(SonarLintTestHarness harness) {
3754
var serverIssue = aServerIssue("myIssueKey").withTextRange(new TextRangeWithHash(1, 2, 3, 4, "hash")).withIntroductionDate(Instant.EPOCH.plusSeconds(1)).withType(RuleType.BUG);
@@ -48,4 +65,23 @@ void it_should_purge_local_only_stored_issues_resolved_more_than_one_week_ago_at
4865

4966
assertThat(storedIssues).isEmpty();
5067
}
68+
69+
@SonarLintTest
70+
void it_should_migrate_the_local_only_issues_from_xodus_to_the_new_h2_database(SonarLintTestHarness harness) {
71+
environmentVariables.set(SONARSOURCE_DOGFOODING_ENV_VAR_KEY, "1");
72+
var serverIssue = aServerIssue("myIssueKey").withTextRange(new TextRangeWithHash(1, 2, 3, 4, "hash")).withIntroductionDate(Instant.EPOCH.plusSeconds(1)).withType(RuleType.BUG);
73+
var backend = harness.newBackend()
74+
.withSonarQubeConnection("connectionId", harness.newFakeSonarQubeServer().start(), storage -> storage
75+
.withProject("projectKey", project -> project.withMainBranch("main", branch -> branch.withIssue(serverIssue)))
76+
.withServerVersion("9.8"))
77+
.withBoundConfigScope("configScopeId", "connectionId", "projectKey",
78+
storage -> storage.noH2()
79+
.withLocalOnlyIssue(aLocalOnlyIssueResolved()))
80+
.start();
81+
var localOnlyIssuesRepository = new LocalOnlyIssuesRepository(backend.getSonarLintDatabase());
82+
83+
var issues = localOnlyIssuesRepository.loadForFile("configScopeId", Paths.get("file/path"));
84+
85+
assertThat(issues).hasSize(1);
86+
}
5187
}

‎test-utils/src/main/java/org/sonarsource/sonarlint/core/test/utils/storage/ConfigurationScopeStorageFixture.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,17 @@ public static ConfigurationScopeStorageBuilder newBuilder(String configScopeId)
4848
public static class ConfigurationScopeStorageBuilder {
4949
private final List<LocalOnlyIssue> localOnlyIssues = new ArrayList<>();
5050
private final String configScopeId;
51+
private boolean noH2;
5152

5253
public ConfigurationScopeStorageBuilder(String configScopeId) {
5354
this.configScopeId = configScopeId;
5455
}
5556

57+
public ConfigurationScopeStorageBuilder noH2() {
58+
this.noH2 = true;
59+
return this;
60+
}
61+
5662
public ConfigurationScopeStorageBuilder withLocalOnlyIssue(LocalOnlyIssue issue) {
5763
localOnlyIssues.add(issue);
5864
return this;
@@ -119,8 +125,10 @@ public void create(Path storageRoot) {
119125
}
120126

121127
public void populateDatabase(SonarLintDatabase database) {
122-
var localOnlyIssuesRepository = new LocalOnlyIssuesRepository(database);
123-
localOnlyIssues.forEach(issue -> localOnlyIssuesRepository.storeLocalOnlyIssue(configScopeId, issue));
128+
if (!noH2) {
129+
var localOnlyIssuesRepository = new LocalOnlyIssuesRepository(database);
130+
localOnlyIssues.forEach(issue -> localOnlyIssuesRepository.storeLocalOnlyIssue(configScopeId, issue));
131+
}
124132
}
125133
}
126134
}

0 commit comments

Comments
 (0)