Skip to content

Commit 11d525a

Browse files
author
amvanbaren
committed
Cache /api/-/search results
# Conflicts: # server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java # server/src/main/resources/ehcache.xml # server/src/test/java/org/eclipse/openvsx/AdminAPITest.java # server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java
1 parent a874a24 commit 11d525a

File tree

11 files changed

+209
-81
lines changed

11 files changed

+209
-81
lines changed

‎server/src/main/java/org/eclipse/openvsx/ExtensionService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ public void updateExtension(Extension extension) {
117117
cache.evictNamespaceDetails(extension);
118118
cache.evictLatestExtensionVersion(extension);
119119
cache.evictExtensionJsons(extension);
120+
cache.evictSearchEntryJsons(extension);
120121

121122
if (extension.getVersions().stream().anyMatch(ExtensionVersion::isActive)) {
122123
// There is at least one active version => activate the extension

‎server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java‎

Lines changed: 18 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.eclipse.openvsx.entities.*;
3030
import org.eclipse.openvsx.json.*;
3131
import org.eclipse.openvsx.repositories.RepositoryService;
32-
import org.eclipse.openvsx.search.ExtensionSearch;
3332
import org.eclipse.openvsx.search.ISearchService;
3433
import org.eclipse.openvsx.search.SearchUtilService;
3534
import org.eclipse.openvsx.storage.StorageUtilService;
@@ -39,8 +38,6 @@
3938
import org.springframework.beans.factory.annotation.Autowired;
4039
import org.springframework.cache.annotation.Cacheable;
4140
import org.springframework.dao.DataIntegrityViolationException;
42-
import org.springframework.data.elasticsearch.core.SearchHit;
43-
import org.springframework.data.elasticsearch.core.SearchHits;
4441
import org.springframework.http.HttpStatus;
4542
import org.springframework.http.ResponseEntity;
4643
import org.springframework.retry.annotation.Retryable;
@@ -82,6 +79,9 @@ public class LocalRegistryService implements IExtensionRegistry {
8279
@Autowired
8380
CacheService cache;
8481

82+
@Autowired
83+
SearchEntryService searchEntries;
84+
8585
@Override
8686
public NamespaceJson getNamespace(String namespaceName) {
8787
var namespace = repositories.findNamespace(namespaceName);
@@ -211,7 +211,21 @@ public SearchResultJson search(ISearchService.Options options) {
211211
}
212212

213213
var searchHits = search.search(options);
214-
json.extensions = toSearchEntries(searchHits, options);
214+
var extensions = new ArrayList<SearchEntryJson>();
215+
for (var searchHit : searchHits) {
216+
var searchEntry = searchEntries.toJson(searchHit, options.includeAllVersions);
217+
if(searchEntry != null) {
218+
// use averageRating and downloadCount from ElasticSearch response,
219+
// so that cached SearchEntryJson doesn't have to be evicted every time
220+
// averageRating or downloadCount are updated.
221+
var extensionSearch = searchHit.getContent();
222+
searchEntry.averageRating = extensionSearch.averageRating;
223+
searchEntry.downloadCount = extensionSearch.downloadCount;
224+
extensions.add(searchEntry);
225+
}
226+
}
227+
228+
json.extensions = extensions;
215229
json.offset = options.requestedOffset;
216230
json.totalSize = (int) searchHits.getTotalHits();
217231
return json;
@@ -741,81 +755,6 @@ public ResultJson deleteReview(String namespace, String extensionName) {
741755
return ResultJson.success("Deleted review for " + extension.getNamespace().getName() + "." + extension.getName());
742756
}
743757

744-
private Extension getExtension(SearchHit<ExtensionSearch> searchHit) {
745-
var searchItem = searchHit.getContent();
746-
var extension = entityManager.find(Extension.class, searchItem.id);
747-
if (extension == null || !extension.isActive()) {
748-
extension = new Extension();
749-
extension.setId(searchItem.id);
750-
search.removeSearchEntry(extension);
751-
return null;
752-
}
753-
754-
return extension;
755-
}
756-
757-
private List<SearchEntryJson> toSearchEntries(SearchHits<ExtensionSearch> searchHits, ISearchService.Options options) {
758-
var serverUrl = UrlUtil.getBaseUrl();
759-
var extensions = searchHits.stream()
760-
.map(this::getExtension)
761-
.filter(Objects::nonNull)
762-
.collect(Collectors.toList());
763-
764-
var latestVersions = extensions.stream()
765-
.map(e -> {
766-
var latest = versions.getLatestTrxn(e, null, false, true);
767-
return new AbstractMap.SimpleEntry<>(e.getId(), latest);
768-
})
769-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
770-
771-
var searchEntries = latestVersions.entrySet().stream()
772-
.map(e -> {
773-
var entry = e.getValue().toSearchEntryJson();
774-
entry.url = createApiUrl(serverUrl, "api", entry.namespace, entry.name);
775-
return new AbstractMap.SimpleEntry<>(e.getKey(), entry);
776-
})
777-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
778-
779-
var fileUrls = storageUtil.getFileUrls(latestVersions.values(), serverUrl, DOWNLOAD, ICON);
780-
searchEntries.forEach((extensionId, searchEntry) -> searchEntry.files = fileUrls.get(latestVersions.get(extensionId).getId()));
781-
if (options.includeAllVersions) {
782-
var allActiveVersions = repositories.findActiveVersions(extensions).stream()
783-
.sorted(ExtensionVersion.SORT_COMPARATOR)
784-
.collect(Collectors.toList());
785-
786-
var activeVersionsByExtensionId = allActiveVersions.stream()
787-
.collect(Collectors.groupingBy(ev -> ev.getExtension().getId()));
788-
789-
var versionUrls = storageUtil.getFileUrls(allActiveVersions, serverUrl, DOWNLOAD);
790-
for(var extension : extensions) {
791-
var activeVersions = activeVersionsByExtensionId.get(extension.getId());
792-
var searchEntry = searchEntries.get(extension.getId());
793-
searchEntry.allVersions = getAllVersionReferences(activeVersions, versionUrls, serverUrl);
794-
}
795-
}
796-
797-
return extensions.stream()
798-
.map(Extension::getId)
799-
.map(searchEntries::get)
800-
.collect(Collectors.toList());
801-
}
802-
803-
private List<SearchEntryJson.VersionReference> getAllVersionReferences(
804-
List<ExtensionVersion> extVersions,
805-
Map<Long, Map<String, String>> versionUrls,
806-
String serverUrl
807-
) {
808-
Collections.sort(extVersions, ExtensionVersion.SORT_COMPARATOR);
809-
return extVersions.stream().map(extVersion -> {
810-
var ref = new SearchEntryJson.VersionReference();
811-
ref.version = extVersion.getVersion();
812-
ref.engines = extVersion.getEnginesMap();
813-
ref.url = UrlUtil.createApiVersionUrl(serverUrl, extVersion);
814-
ref.files = versionUrls.get(extVersion.getId());
815-
return ref;
816-
}).collect(Collectors.toList());
817-
}
818-
819758
public ExtensionJson toExtensionVersionJson(ExtensionVersion extVersion, String targetPlatform, boolean onlyActive, boolean inTransaction) {
820759
var extension = extVersion.getExtension();
821760
var latest = inTransaction

‎server/src/main/java/org/eclipse/openvsx/cache/CacheService.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,13 @@ public class CacheService {
2727

2828
public static final String CACHE_DATABASE_SEARCH = "database.search";
2929
public static final String CACHE_EXTENSION_JSON = "extension.json";
30+
public static final String CACHE_SEARCH_ENTRY_JSON = "search.entry.json";
3031
public static final String CACHE_LATEST_EXTENSION_VERSION = "latest.extension.version";
3132
public static final String CACHE_NAMESPACE_DETAILS_JSON = "namespace.details.json";
3233
public static final String CACHE_AVERAGE_REVIEW_RATING = "average.review.rating";
3334

3435
public static final String GENERATOR_EXTENSION_JSON = "extensionJsonCacheKeyGenerator";
36+
public static final String GENERATOR_SEARCH_ENTRY_JSON = "searchEntryJsonCacheKeyGenerator";
3537
public static final String GENERATOR_LATEST_EXTENSION_VERSION = "latestExtensionVersionCacheKeyGenerator";
3638

3739
@Autowired
@@ -43,6 +45,9 @@ public class CacheService {
4345
@Autowired
4446
ExtensionJsonCacheKeyGenerator extensionJsonCacheKey;
4547

48+
@Autowired
49+
SearchEntryJsonCacheKeyGenerator searchEntryJsonCacheKeyGenerator;
50+
4651
@Autowired
4752
LatestExtensionVersionCacheKeyGenerator latestExtensionVersionCacheKey;
4853

@@ -92,6 +97,19 @@ public void evictExtensionJsons(Extension extension) {
9297
}
9398
}
9499

100+
public void evictSearchEntryJsons(Extension extension) {
101+
var cache = cacheManager.getCache(CACHE_SEARCH_ENTRY_JSON);
102+
if(cache == null) {
103+
return; // cache is not created
104+
}
105+
106+
var includeAllVersionsList = List.of(true, false);
107+
for(var includeAllVersions : includeAllVersionsList) {
108+
var key = searchEntryJsonCacheKeyGenerator.generate(extension.getId(), includeAllVersions);
109+
cache.evictIfPresent(key);
110+
}
111+
}
112+
95113
public void evictLatestExtensionVersion(Extension extension) {
96114
var cache = cacheManager.getCache(CACHE_LATEST_EXTENSION_VERSION);
97115
if(cache != null) {

‎server/src/main/java/org/eclipse/openvsx/cache/SearchEntryJsonCacheKeyGenerator.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/** ******************************************************************************
2+
* Copyright (c) 2022 Precies. Software Ltd and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0.
7+
*
8+
* SPDX-License-Identifier: EPL-2.0
9+
* ****************************************************************************** */
10+
package org.eclipse.openvsx.cache;
11+
12+
import org.eclipse.openvsx.search.ExtensionSearch;
13+
import org.springframework.cache.interceptor.KeyGenerator;
14+
import org.springframework.data.elasticsearch.core.SearchHit;
15+
import org.springframework.stereotype.Component;
16+
17+
import java.lang.reflect.Method;
18+
19+
@Component
20+
public class SearchEntryJsonCacheKeyGenerator implements KeyGenerator {
21+
22+
@Override
23+
public Object generate(Object target, Method method, Object... params) {
24+
var searchHit = (SearchHit<?>) params[0];
25+
var includeAllVersions = (boolean) params[1];
26+
if(searchHit.getContent() instanceof ExtensionSearch) {
27+
var extensionSearch = (ExtensionSearch) searchHit.getContent();
28+
return generate(extensionSearch.id, includeAllVersions);
29+
}
30+
31+
return null;
32+
}
33+
34+
public Object generate(long extensionId, boolean includeAllVersions) {
35+
return "extensionId=" + extensionId + ",includeAllVersions=" + includeAllVersions;
36+
}
37+
}

‎server/src/main/java/org/eclipse/openvsx/entities/Extension.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public ExtensionSearch toSearch(ExtensionVersion latest) {
6262
search.name = this.getName();
6363
search.namespace = this.getNamespace().getName();
6464
search.extensionId = search.namespace + "." + search.name;
65+
search.averageRating = this.getAverageRating();
6566
search.downloadCount = this.getDownloadCount();
6667
search.targetPlatforms = this.getVersions().stream()
6768
.map(ExtensionVersion::getTargetPlatform)

‎server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public class SearchEntryJson implements Serializable {
7878
name = "VersionReference",
7979
description = "Essential metadata of an extension version"
8080
)
81-
public static class VersionReference {
81+
public static class VersionReference implements Serializable {
8282

8383
@Schema(description = "URL to get the full metadata of this version")
8484
public String url;

‎server/src/main/java/org/eclipse/openvsx/json/SearchEntryService.java‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/** ******************************************************************************
2+
* Copyright (c) 2022 Precies. Software Ltd and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0.
7+
*
8+
* SPDX-License-Identifier: EPL-2.0
9+
* ****************************************************************************** */
10+
package org.eclipse.openvsx.json;
11+
12+
import org.eclipse.openvsx.entities.Extension;
13+
import org.eclipse.openvsx.entities.ExtensionVersion;
14+
import org.eclipse.openvsx.repositories.RepositoryService;
15+
import org.eclipse.openvsx.search.ExtensionSearch;
16+
import org.eclipse.openvsx.search.SearchUtilService;
17+
import org.eclipse.openvsx.storage.StorageUtilService;
18+
import org.eclipse.openvsx.util.UrlUtil;
19+
import org.eclipse.openvsx.util.VersionService;
20+
import org.springframework.beans.factory.annotation.Autowired;
21+
import org.springframework.cache.annotation.Cacheable;
22+
import org.springframework.data.elasticsearch.core.SearchHit;
23+
import org.springframework.stereotype.Component;
24+
25+
import javax.persistence.EntityManager;
26+
import javax.transaction.Transactional;
27+
import java.util.*;
28+
import java.util.stream.Collectors;
29+
30+
import static org.eclipse.openvsx.cache.CacheService.CACHE_SEARCH_ENTRY_JSON;
31+
import static org.eclipse.openvsx.cache.CacheService.GENERATOR_SEARCH_ENTRY_JSON;
32+
import static org.eclipse.openvsx.entities.FileResource.DOWNLOAD;
33+
import static org.eclipse.openvsx.entities.FileResource.ICON;
34+
import static org.eclipse.openvsx.util.UrlUtil.createApiUrl;
35+
36+
@Component
37+
public class SearchEntryService {
38+
39+
@Autowired
40+
EntityManager entityManager;
41+
42+
@Autowired
43+
VersionService versions;
44+
45+
@Autowired
46+
StorageUtilService storageUtil;
47+
48+
@Autowired
49+
SearchUtilService search;
50+
51+
@Autowired
52+
RepositoryService repositories;
53+
54+
@Transactional
55+
@Cacheable(value = CACHE_SEARCH_ENTRY_JSON, keyGenerator = GENERATOR_SEARCH_ENTRY_JSON)
56+
public SearchEntryJson toJson(SearchHit<ExtensionSearch> searchHit, boolean includeAllVersions) {
57+
var serverUrl = UrlUtil.getBaseUrl();
58+
var extension = getExtension(searchHit);
59+
if(extension == null) {
60+
return null;
61+
}
62+
63+
var latest = versions.getLatest(extension, null, false, true);
64+
var searchEntry = latest.toSearchEntryJson();
65+
searchEntry.url = createApiUrl(serverUrl, "api", searchEntry.namespace, searchEntry.name);
66+
searchEntry.files = storageUtil.getFileUrls(latest, serverUrl, DOWNLOAD, ICON);
67+
68+
if (includeAllVersions) {
69+
var activeVersions = repositories.findActiveVersions(extension).toList();
70+
var versionUrls = storageUtil.getFileUrls(activeVersions, serverUrl, DOWNLOAD);
71+
searchEntry.allVersions = getAllVersionReferences(activeVersions, versionUrls, serverUrl);
72+
}
73+
74+
return searchEntry;
75+
}
76+
77+
private Extension getExtension(SearchHit<ExtensionSearch> searchHit) {
78+
var searchItem = searchHit.getContent();
79+
var extension = entityManager.find(Extension.class, searchItem.id);
80+
if (extension == null || !extension.isActive()) {
81+
extension = new Extension();
82+
extension.setId(searchItem.id);
83+
search.removeSearchEntry(extension);
84+
return null;
85+
}
86+
87+
return extension;
88+
}
89+
90+
private List<SearchEntryJson.VersionReference> getAllVersionReferences(
91+
List<ExtensionVersion> extVersions,
92+
Map<Long, Map<String, String>> versionUrls,
93+
String serverUrl
94+
) {
95+
return extVersions.stream()
96+
.sorted(ExtensionVersion.SORT_COMPARATOR)
97+
.map(extVersion -> {
98+
var ref = new SearchEntryJson.VersionReference();
99+
ref.version = extVersion.getVersion();
100+
ref.engines = extVersion.getEnginesMap();
101+
ref.url = UrlUtil.createApiVersionUrl(serverUrl, extVersion);
102+
ref.files = versionUrls.get(extVersion.getId());
103+
return ref;
104+
}).collect(Collectors.toList());
105+
}
106+
}

‎server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ public class ExtensionSearch implements Serializable {
4444
@Field(index = false)
4545
public long timestamp;
4646

47+
@Nullable
48+
@Field(index = false, type = FieldType.Float)
49+
public Double averageRating;
50+
4751
@Nullable
4852
@Field(index = false, type = FieldType.Float)
4953
public Double rating;

‎server/src/main/resources/ehcache.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,16 @@
3030
<heap unit="entries">1024</heap>
3131
</resources>
3232
</cache>
33+
<cache alias="search.entry.json">
34+
<expiry>
35+
<ttl unit="seconds">3600</ttl>
36+
</expiry>
37+
<resources>
38+
<heap unit="entries">1024</heap>
39+
<offheap unit="MB">32</offheap>
40+
<disk unit="MB">128</disk>
41+
</resources>
42+
</cache>
3343
<cache alias="extension.json">
3444
<expiry>
3545
<ttl unit="seconds">3600</ttl>

‎server/src/test/java/org/eclipse/openvsx/AdminAPITest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.eclipse.openvsx.storage.StorageUtilService;
3030
import org.eclipse.openvsx.util.TargetPlatform;
3131
import org.eclipse.openvsx.util.VersionService;
32+
import org.jobrunr.scheduling.JobRequestScheduler;
3233
import org.junit.jupiter.api.Test;
3334
import org.mockito.Mockito;
3435
import org.springframework.beans.factory.annotation.Autowired;
@@ -67,7 +68,7 @@
6768
ClientRegistrationRepository.class, UpstreamRegistryService.class, GoogleCloudStorageService.class,
6869
AzureBlobStorageService.class, VSCodeIdService.class, AzureDownloadCountService.class,
6970
CacheService.class, PublishExtensionVersionHandler.class, SearchUtilService.class,
70-
EclipseService.class, SimpleMeterRegistry.class
71+
EclipseService.class, SimpleMeterRegistry.class, SearchEntryService.class
7172
})
7273
public class AdminAPITest {
7374

0 commit comments

Comments
 (0)