From c559b7d9d46312617bc4558482d6ab2a6a1aebb9 Mon Sep 17 00:00:00 2001 From: yiningwang11 Date: Thu, 2 Feb 2023 16:49:25 -0500 Subject: [PATCH 01/45] fix issue 613 --- server/src/main/java/org/eclipse/openvsx/RegistryAPI.java | 1 - webui/src/pages/extension-detail/extension-review-dialog.tsx | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java index 29bc781d4..decb81227 100644 --- a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java @@ -632,7 +632,6 @@ public ResponseEntity getReviews( for (var registry : getRegistries()) { try { return ResponseEntity.ok() - .cacheControl(CacheControl.maxAge(10, TimeUnit.MINUTES).cachePublic()) .body(registry.getReviews(namespace, extension)); } catch (NotFoundException exc) { // Try the next registry diff --git a/webui/src/pages/extension-detail/extension-review-dialog.tsx b/webui/src/pages/extension-detail/extension-review-dialog.tsx index 1488aecaf..cea2e7a05 100644 --- a/webui/src/pages/extension-detail/extension-review-dialog.tsx +++ b/webui/src/pages/extension-detail/extension-review-dialog.tsx @@ -109,9 +109,9 @@ class ExtensionReviewDialogComponent extends React.Component - + )} {this.props.extension.displayName || this.props.extension.name} Review From 5525f1b6684fe9f0387c71563b98371f3b5b6011 Mon Sep 17 00:00:00 2001 From: yiningwang11 Date: Tue, 28 Feb 2023 12:15:45 -0500 Subject: [PATCH 02/45] adding etag for getReviews rebase branch "updateReview" onto branch "master" --- server/src/main/java/org/eclipse/openvsx/RegistryAPI.java | 1 + .../src/main/java/org/eclipse/openvsx/cache/CacheService.java | 2 ++ .../java/org/eclipse/openvsx/web/ShallowEtagHeaderFilter.java | 4 ---- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java index decb81227..2bd7ed27d 100644 --- a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java @@ -632,6 +632,7 @@ public ResponseEntity getReviews( for (var registry : getRegistries()) { try { return ResponseEntity.ok() + .cacheControl(CacheControl.noCache().cachePublic()) .body(registry.getReviews(namespace, extension)); } catch (NotFoundException exc) { // Try the next registry diff --git a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java index d22e37067..8cb34c6d0 100644 --- a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java +++ b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java @@ -12,9 +12,11 @@ import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.UserData; +import org.eclipse.openvsx.json.ReviewListJson; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.util.TargetPlatform; import org.eclipse.openvsx.util.VersionAlias; +import org.eclipse.openvsx.util.UrlUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.CacheManager; import org.springframework.stereotype.Component; diff --git a/server/src/main/java/org/eclipse/openvsx/web/ShallowEtagHeaderFilter.java b/server/src/main/java/org/eclipse/openvsx/web/ShallowEtagHeaderFilter.java index f4e48560c..ae2ad1e5d 100644 --- a/server/src/main/java/org/eclipse/openvsx/web/ShallowEtagHeaderFilter.java +++ b/server/src/main/java/org/eclipse/openvsx/web/ShallowEtagHeaderFilter.java @@ -22,10 +22,6 @@ protected boolean shouldNotFilter(HttpServletRequest request) throws ServletExce if(applyFilter) { applyFilter = path[0].equals("api") && !path[1].equals("-"); } - if(applyFilter && path.length == 4) { - applyFilter = !(path[3].equals("review") || path[3].equals("reviews")); - } - return !applyFilter; } } From 6fe5bd92b917a4175f4bf24084d653d263351ea6 Mon Sep 17 00:00:00 2001 From: yiningwang11 Date: Tue, 28 Feb 2023 12:22:54 -0500 Subject: [PATCH 03/45] unmodify CacheService.java --- server/src/main/java/org/eclipse/openvsx/cache/CacheService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java index 8cb34c6d0..c898d05df 100644 --- a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java +++ b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java @@ -12,7 +12,6 @@ import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.UserData; -import org.eclipse.openvsx.json.ReviewListJson; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.util.TargetPlatform; import org.eclipse.openvsx.util.VersionAlias; From f816b46af136b9fc1dbc21bd38ed5e206756e3fe Mon Sep 17 00:00:00 2001 From: yiningwang11 Date: Tue, 14 Mar 2023 10:44:26 -0400 Subject: [PATCH 04/45] Update average rating according to review updates --- .../src/main/java/org/eclipse/openvsx/LocalRegistryService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java index 7647440ac..a7d101874 100644 --- a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java @@ -111,7 +111,9 @@ public ExtensionJson getExtension(String namespace, String extensionName, String public ExtensionJson getExtension(String namespace, String extensionName, String targetPlatform, String version) { var extVersion = findExtensionVersion(namespace, extensionName, targetPlatform, version); var json = toExtensionVersionJson(extVersion, targetPlatform, true, false); + var extension = repositories.findExtension(extensionName, namespace); json.downloads = getDownloads(extVersion.getExtension(), targetPlatform, extVersion.getVersion()); + json.averageRating = extension.getAverageRating(); return json; } From bd5ae3d9703346d5e85a65191b925bd09c5f1380 Mon Sep 17 00:00:00 2001 From: yiningwang11 Date: Thu, 23 Mar 2023 11:24:02 -0400 Subject: [PATCH 05/45] update ratings on homepage --- .../java/org/eclipse/openvsx/LocalRegistryService.java | 1 + server/src/main/java/org/eclipse/openvsx/RegistryAPI.java | 2 +- .../org/eclipse/openvsx/web/ShallowEtagHeaderFilter.java | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java index a7d101874..ee02d7983 100644 --- a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java @@ -778,6 +778,7 @@ private List toSearchEntries(SearchHits search .map(e -> { var entry = e.getValue().toSearchEntryJson(); entry.url = createApiUrl(serverUrl, "api", entry.namespace, entry.name); + entry.averageRating = repositories.findExtension(entry.name, entry.namespace).getAverageRating(); return new AbstractMap.SimpleEntry<>(e.getKey(), entry); }) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); diff --git a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java index 2bd7ed27d..3bba8b244 100644 --- a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java @@ -750,7 +750,7 @@ public ResponseEntity search( } return ResponseEntity.ok() - .cacheControl(CacheControl.maxAge(10, TimeUnit.MINUTES).cachePublic()) + .cacheControl(CacheControl.noCache().cachePublic()) .body(result); } diff --git a/server/src/main/java/org/eclipse/openvsx/web/ShallowEtagHeaderFilter.java b/server/src/main/java/org/eclipse/openvsx/web/ShallowEtagHeaderFilter.java index ae2ad1e5d..915fd3ae9 100644 --- a/server/src/main/java/org/eclipse/openvsx/web/ShallowEtagHeaderFilter.java +++ b/server/src/main/java/org/eclipse/openvsx/web/ShallowEtagHeaderFilter.java @@ -16,11 +16,11 @@ public class ShallowEtagHeaderFilter extends org.springframework.web.filter.Shal protected boolean shouldNotFilter(HttpServletRequest request) throws ServletException { // limit the filter to /api/{namespace}/{extension}, /api/{namespace}/details - // and /api/{namespace}/{extension}/{version} endpoints + // and /api/{namespace}/{extension}/{version}, and /api/-/search endpoints var path = request.getRequestURI().substring(1).split("/"); - var applyFilter = path.length == 3 || path.length == 4; - if(applyFilter) { - applyFilter = path[0].equals("api") && !path[1].equals("-"); + var applyFilter = (path.length == 3 || path.length == 4) && path[0].equals("api"); + if(applyFilter && path[1].equals("-")) { + applyFilter = path[2].contains("search"); } return !applyFilter; } From a6a9e61559c3f306e10911b49db6c7ea75da58c2 Mon Sep 17 00:00:00 2001 From: Jean Pierre Date: Tue, 31 Jan 2023 14:22:11 +0000 Subject: [PATCH 06/45] Map.of doesnt allow null values --- .../openvsx/UpstreamRegistryService.java | 66 +++++++++---------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java b/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java index b6e999f04..0333760b2 100644 --- a/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java @@ -163,7 +163,7 @@ private ResponseEntity getFile(String urlTemplate, Map uriVar var url = UriComponentsBuilder.fromUriString(urlTemplate).build(uriVariables); logger.error("HEAD " + url, exc); } - + throw new NotFoundException(); } var statusCode = response.getStatusCode(); @@ -196,7 +196,7 @@ public ReviewListJson getReviews(String namespace, String extension) { var url = UriComponentsBuilder.fromUriString(urlTemplate).build(uriVariables); logger.error("GET " + url, exc); } - + throw new NotFoundException(); } } @@ -204,16 +204,15 @@ public ReviewListJson getReviews(String namespace, String extension) { @Override public SearchResultJson search(ISearchService.Options options) { var urlTemplate = urlConfigService.getUpstreamUrl() + "/api/-/search"; - var uriVariables = Map.of( - "size", Integer.toString(options.requestedSize), - "offset", Integer.toString(options.requestedOffset), - "includeAllVersions", Boolean.toString(options.includeAllVersions), - "query", options.queryString, - "category", options.category, - "sortOrder", options.sortOrder, - "sortBy", options.sortBy, - "targetPlatform", options.targetPlatform - ); + var uriVariables = new HashMap(); + uriVariables.put("size", Integer.toString(options.requestedSize)); + uriVariables.put("offset", Integer.toString(options.requestedOffset)); + uriVariables.put("includeAllVersions", Boolean.toString(options.includeAllVersions)); + uriVariables.put("query", options.queryString); + uriVariables.put("category", options.category); + uriVariables.put("sortOrder", options.sortOrder); + uriVariables.put("sortBy", options.sortBy); + uriVariables.put("targetPlatform", options.targetPlatform); var queryString = uriVariables.entrySet().stream() .filter(entry -> !Strings.isNullOrEmpty(entry.getValue())) @@ -238,16 +237,16 @@ public SearchResultJson search(ISearchService.Options options) { @Override public QueryResultJson query(QueryParamJson param) { var urlTemplate = urlConfigService.getUpstreamUrl() + "/api/-/query"; - var queryParams = Map.of( - "namespaceName", param.namespaceName, - "extensionName", param.extensionName, - "extensionVersion", param.extensionVersion, - "extensionId", param.extensionId, - "extensionUuid", param.extensionUuid, - "namespaceUuid", param.namespaceUuid, - "includeAllVersions", String.valueOf(param.includeAllVersions), - "targetPlatform", param.targetPlatform - ); + var queryParams = new HashMap(); + queryParams.put("namespaceName", param.namespaceName); + queryParams.put("extensionName", param.extensionName); + queryParams.put("extensionVersion", param.extensionVersion); + queryParams.put("extensionId", param.extensionId); + queryParams.put("extensionUuid", param.extensionUuid); + queryParams.put("namespaceUuid", param.namespaceUuid); + queryParams.put("includeAllVersions", String.valueOf(param.includeAllVersions)); + queryParams.put("targetPlatform", param.targetPlatform); + var queryString = queryParams.entrySet().stream() .filter(entry -> !Strings.isNullOrEmpty(entry.getValue())) @@ -272,16 +271,15 @@ public QueryResultJson query(QueryParamJson param) { @Override public QueryResultJson queryV2(QueryParamJsonV2 param) { var urlTemplate = urlConfigService.getUpstreamUrl() + "/api/v2/-/query"; - var queryParams = Map.of( - "namespaceName", param.namespaceName, - "extensionName", param.extensionName, - "extensionVersion", param.extensionVersion, - "extensionId", param.extensionId, - "extensionUuid", param.extensionUuid, - "namespaceUuid", param.namespaceUuid, - "includeAllVersions", String.valueOf(param.includeAllVersions), - "targetPlatform", param.targetPlatform - ); + var queryParams = new HashMap(); + queryParams.put("namespaceName", param.namespaceName); + queryParams.put("extensionName", param.extensionName); + queryParams.put("extensionVersion", param.extensionVersion); + queryParams.put("extensionId", param.extensionId); + queryParams.put("extensionUuid", param.extensionUuid); + queryParams.put("namespaceUuid", param.namespaceUuid); + queryParams.put("includeAllVersions", String.valueOf(param.includeAllVersions)); + queryParams.put("targetPlatform", param.targetPlatform); var queryString = queryParams.entrySet().stream() .filter(entry -> !Strings.isNullOrEmpty(entry.getValue())) @@ -302,7 +300,7 @@ public QueryResultJson queryV2(QueryParamJsonV2 param) { throw new NotFoundException(); } } - + private void handleError(Throwable exc) throws RuntimeException { if (exc instanceof HttpStatusCodeException) { var status = ((HttpStatusCodeException) exc).getStatusCode(); @@ -327,4 +325,4 @@ private boolean isNotFound(RestClientException exc) { return exc instanceof HttpStatusCodeException && ((HttpStatusCodeException) exc).getStatusCode() == HttpStatus.NOT_FOUND; } -} \ No newline at end of file +} From 4b5e15034389ca1ac9ccf5b75f579a2c2907e0b4 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Thu, 2 Feb 2023 10:15:49 +0200 Subject: [PATCH 07/45] IMDB rating --- ...stryAPIGetNamespaceDetailsSimulation.scala | 18 ++++++ .../scala/org/eclipse/openvsx/Scenarios.scala | 14 ++++- .../src/gatling/scripts/test-registry-api.sh | 1 + .../eclipse/openvsx/LocalRegistryService.java | 48 +++++++------- .../openvsx/adapter/LocalVSCodeService.java | 14 +---- .../eclipse/openvsx/cache/CacheService.java | 5 +- .../cache/ExtensionJsonCacheKeyGenerator.java | 3 +- .../eclipse/openvsx/entities/Extension.java | 15 ++++- .../openvsx/entities/ExtensionVersion.java | 1 + .../eclipse/openvsx/json/SearchEntryJson.java | 4 ++ .../openvsx/mirror/DataMirrorService.java | 1 + .../repositories/ExtensionJooqRepository.java | 16 +---- .../ExtensionReviewRepository.java | 10 +++ .../ExtensionVersionJooqRepository.java | 2 + .../repositories/RepositoryService.java | 12 ++-- .../openvsx/search/DatabaseSearchService.java | 62 +++++++++---------- .../openvsx/search/ElasticSearchService.java | 11 +++- .../openvsx/search/ExtensionSearch.java | 6 +- .../openvsx/search/RelevanceService.java | 35 ++++++++--- .../storage/AzureDownloadCountProcessor.java | 3 +- .../storage/AzureDownloadCountService.java | 2 +- .../openvsx/storage/StorageUtilService.java | 2 +- .../openvsx/jooq/tables/Extension.java | 13 ++-- .../jooq/tables/records/ExtensionRecord.java | 57 ++++++++++++++--- .../V1_33__Extension_ReviewCount.sql | 11 ++++ server/src/main/resources/ehcache.xml | 20 +++--- .../org/eclipse/openvsx/RegistryAPITest.java | 4 -- .../openvsx/adapter/VSCodeAPITest.java | 5 +- .../openvsx/cache/CacheServiceTest.java | 33 +++++++++- .../RepositoryServiceSmokeTest.java | 5 +- .../search/DatabaseSearchServiceTest.java | 16 ++--- .../search/ElasticSearchServiceTest.java | 6 +- webui/src/extension-registry-types.ts | 3 +- .../extension-list/extension-list-header.tsx | 2 +- .../extension-list/extension-list-item.tsx | 3 + 35 files changed, 302 insertions(+), 161 deletions(-) create mode 100644 server/src/gatling/scala/org/eclipse/openvsx/RegistryAPIGetNamespaceDetailsSimulation.scala create mode 100644 server/src/main/resources/db/migration/V1_33__Extension_ReviewCount.sql diff --git a/server/src/gatling/scala/org/eclipse/openvsx/RegistryAPIGetNamespaceDetailsSimulation.scala b/server/src/gatling/scala/org/eclipse/openvsx/RegistryAPIGetNamespaceDetailsSimulation.scala new file mode 100644 index 000000000..621496eef --- /dev/null +++ b/server/src/gatling/scala/org/eclipse/openvsx/RegistryAPIGetNamespaceDetailsSimulation.scala @@ -0,0 +1,18 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx + +import io.gatling.core.Predef._ + +import org.eclipse.openvsx.Scenarios._ + +class RegistryAPIGetNamespaceDetailsSimulation extends Simulation { + setUp(getNamespaceDetailsScenario().inject(atOnceUsers(users))).protocols(httpProtocol) +} \ No newline at end of file diff --git a/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala b/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala index 6fab0591f..964ff9f1e 100644 --- a/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala +++ b/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala @@ -16,7 +16,6 @@ import io.gatling.core.structure.ScenarioBuilder import io.gatling.http.Predef._ import java.nio.file.Files -import scala.:: import scala.collection.mutable.ListBuffer import scala.concurrent.duration.DurationInt import scala.reflect.io.File @@ -31,7 +30,7 @@ object Scenarios { private def headers(): Map[String,String] = { var headers: Map[String,String] = Map() if(conf.hasPath("auth")) { - headers += "Authorization" -> conf.getString("auth"); + headers += "Authorization" -> conf.getString("auth") } headers @@ -195,6 +194,17 @@ object Scenarios { } } + def getNamespaceDetailsScenario(): ScenarioBuilder = { + scenario("RegistryAPI: Get Namespace Details") + .repeat(1000) { + feed(csv("namespaces.csv").circular) + .exec(http("RegistryAPI.getNamespaceDetails") + .get("""/api/${namespace}/details""") + .headers(headers()) + .check(status.is(200))) + } + } + def getQueryScenario(): ScenarioBuilder = { scenario("RegistryAPI: Query") .repeat(1000) { diff --git a/server/src/gatling/scripts/test-registry-api.sh b/server/src/gatling/scripts/test-registry-api.sh index b70ff7977..35eae7e3e 100644 --- a/server/src/gatling/scripts/test-registry-api.sh +++ b/server/src/gatling/scripts/test-registry-api.sh @@ -2,6 +2,7 @@ cd ../../.. ./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetNamespaceSimulation +./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetNamespaceDetailsSimulation ./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetExtensionSimulation ./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetExtensionTargetPlatformSimulation ./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetExtensionVersionSimulation diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java index ee02d7983..f92872d68 100644 --- a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java @@ -41,7 +41,6 @@ import org.springframework.dao.DataIntegrityViolationException; import org.springframework.data.elasticsearch.core.SearchHit; import org.springframework.data.elasticsearch.core.SearchHits; -import org.springframework.http.CacheControl; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.retry.annotation.Retryable; @@ -102,6 +101,7 @@ public NamespaceJson getNamespace(String namespaceName) { } @Override + @Cacheable(value = CACHE_EXTENSION_JSON, keyGenerator = GENERATOR_EXTENSION_JSON) public ExtensionJson getExtension(String namespace, String extensionName, String targetPlatform) { return getExtension(namespace, extensionName, targetPlatform, "latest"); } @@ -172,7 +172,6 @@ private ExtensionVersion findExtensionVersion(String namespace, String extension } @Override - @Transactional public ResponseEntity getFile(String namespace, String extensionName, String targetPlatform, String version, String fileName) { var extVersion = findExtensionVersion(namespace, extensionName, targetPlatform, version); var resource = isType(fileName) ? repositories.findFileByType(extVersion, fileName.toLowerCase()) : repositories.findFileByName(extVersion, fileName); @@ -269,7 +268,7 @@ public QueryResultJson query(QueryParamJson param) { .map(Extension::getId) .collect(Collectors.toSet()); - var reviewCounts = getReviewCounts(extensionIds); + var reviewCounts = getReviewCounts(extensionVersions); var versionStrings = getVersionStrings(extensionVersions); var latestVersions = getLatestVersions(extensionVersions); var latestPreReleases = getLatestVersions(extensionVersions, true); @@ -298,7 +297,7 @@ public QueryResultJson query(QueryParamJson param) { .map(ev -> { var latest = latestVersions.get(getLatestVersionKey(ev)); var latestPreRelease = latestPreReleases.get(getLatestVersionKey(ev)); - var reviewCount = reviewCounts.getOrDefault(ev.getExtension().getId(), 0); + var reviewCount = reviewCounts.getOrDefault(ev.getExtension().getId(), 0L); var preview = previewsByExtensionId.get(ev.getExtension().getId()); var versions = versionStrings.get(ev.getExtension().getId()); var fileResources = fileResourcesByExtensionVersionId.getOrDefault(ev.getId(), Collections.emptyList()); @@ -357,7 +356,7 @@ public QueryResultJson queryV2(QueryParamJsonV2 param) { .map(ev -> ev.getExtension().getId()) .collect(Collectors.toSet()); - var reviewCounts = getReviewCounts(extensionIds); + var reviewCounts = getReviewCounts(extensionVersions); var versionStrings = getVersionStrings(extensionVersions); var latestVersions = getLatestVersions(extensionVersions); var latestPreReleases = getLatestVersions(extensionVersions, true); @@ -393,7 +392,7 @@ public QueryResultJson queryV2(QueryParamJsonV2 param) { .map(ev -> { var latest = latestVersions.get(getLatestVersionKey(ev)); var latestPreRelease = latestPreReleases.get(getLatestVersionKey(ev)); - var reviewCount = reviewCounts.getOrDefault(ev.getExtension().getId(), 0); + var reviewCount = reviewCounts.getOrDefault(ev.getExtension().getId(), 0L); var preview = previewsByExtensionId.get(ev.getExtension().getId()); var globalLatest = addAllVersions ? latestGlobalVersions.get(ev.getExtension().getId()) : null; var globalLatestPreRelease = addAllVersions ? latestGlobalPreReleases.get(ev.getExtension().getId()) : null; @@ -452,10 +451,15 @@ public ResponseEntity getNamespaceLogo(String namespaceName, String file return storageUtil.getNamespaceLogo(namespace); } - private Map getReviewCounts(Collection extensionIds) { - return !extensionIds.isEmpty() - ? repositories.findActiveReviewCountsByExtensionId(extensionIds) - : Collections.emptyMap(); + private Map getReviewCounts(List extensionVersions) { + if(extensionVersions.isEmpty()) { + return Collections.emptyMap(); + } + + return extensionVersions.stream() + .map(ExtensionVersion::getExtension) + .filter(e -> e.getReviewCount() != null) + .collect(Collectors.toMap(Extension::getId, Extension::getReviewCount, (reviews1, reviews2) -> reviews1)); } private Map> getVersionStrings(List extensionVersions) { @@ -704,9 +708,11 @@ public ResultJson postReview(ReviewJson review, String namespace, String extensi extReview.setComment(review.comment); extReview.setRating(review.rating); entityManager.persist(extReview); - extension.setAverageRating(computeAverageRating(extension)); + extension.setAverageRating(repositories.getAverageReviewRating(extension)); + extension.setReviewCount(repositories.countActiveReviews(extension)); search.updateSearchEntry(extension); cache.evictExtensionJsons(extension); + cache.evictLatestExtensionVersion(extension); return ResultJson.success("Added review for " + extension.getNamespace().getName() + "." + extension.getName()); } @@ -728,25 +734,15 @@ public ResultJson deleteReview(String namespace, String extensionName) { for (var extReview : activeReviews) { extReview.setActive(false); } - extension.setAverageRating(computeAverageRating(extension)); + + extension.setAverageRating(repositories.getAverageReviewRating(extension)); + extension.setReviewCount(repositories.countActiveReviews(extension)); search.updateSearchEntry(extension); cache.evictExtensionJsons(extension); + cache.evictLatestExtensionVersion(extension); return ResultJson.success("Deleted review for " + extension.getNamespace().getName() + "." + extension.getName()); } - private Double computeAverageRating(Extension extension) { - var activeReviews = repositories.findActiveReviews(extension).toList(); - if (activeReviews.isEmpty()) { - return null; - } - - var sum = activeReviews.stream() - .mapToLong(ExtensionReview::getRating) - .sum(); - - return (double) sum / activeReviews.size(); - } - private Extension getExtension(SearchHit searchHit) { var searchItem = searchHit.getContent(); var extension = entityManager.find(Extension.class, searchItem.id); @@ -842,7 +838,7 @@ public ExtensionJson toExtensionVersionJson(ExtensionVersion extVersion, String json.verified = isVerified(extVersion); json.namespaceAccess = "restricted"; json.unrelatedPublisher = !json.verified; - json.reviewCount = repositories.countActiveReviews(extension); + json.reviewCount = Optional.ofNullable(extension.getReviewCount()).orElse(0L); var serverUrl = UrlUtil.getBaseUrl(); json.namespaceUrl = createApiUrl(serverUrl, "api", json.namespace); json.reviewsUrl = createApiUrl(serverUrl, "api", json.namespace, json.name, "reviews"); diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java b/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java index 6cd9c267c..dfe519cf7 100644 --- a/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java +++ b/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java @@ -190,14 +190,6 @@ public ExtensionQueryResult extensionQuery(ExtensionQueryParam param, int defaul fileResources = Collections.emptyMap(); } - Map activeReviewCounts; - if(test(flags, FLAG_INCLUDE_STATISTICS) && !extensionsList.isEmpty()) { - var ids = extensionsList.stream().map(Extension::getId).collect(Collectors.toList()); - activeReviewCounts = repositories.findActiveReviewCountsByExtensionId(ids); - } else { - activeReviewCounts = Collections.emptyMap(); - } - var latestVersions = allActiveExtensionVersions.stream() .collect(Collectors.groupingBy(ev -> ev.getExtension().getId())) .values() @@ -208,7 +200,7 @@ public ExtensionQueryResult extensionQuery(ExtensionQueryParam param, int defaul var extensionQueryResults = new ArrayList(); for(var extension : extensionsList) { var latest = latestVersions.get(extension.getId()); - var queryExt = toQueryExtension(extension, latest, activeReviewCounts, flags); + var queryExt = toQueryExtension(extension, latest, flags); queryExt.versions = extensionVersionsMap.getOrDefault(extension.getId(), Collections.emptyList()).stream() .map(extVer -> toQueryVersion(extVer, fileResources, flags)) .collect(Collectors.toList()); @@ -477,7 +469,7 @@ private ResponseEntity browseDirectory( .body(json.getBytes(StandardCharsets.UTF_8)); } - private ExtensionQueryResult.Extension toQueryExtension(Extension extension, ExtensionVersion latest, Map activeReviewCounts, int flags) { + private ExtensionQueryResult.Extension toQueryExtension(Extension extension, ExtensionVersion latest, int flags) { var namespace = extension.getNamespace(); var queryExt = new ExtensionQueryResult.Extension(); @@ -510,7 +502,7 @@ private ExtensionQueryResult.Extension toQueryExtension(Extension extension, Ext } var ratingCountStat = new ExtensionQueryResult.Statistic(); ratingCountStat.statisticName = STAT_RATING_COUNT; - ratingCountStat.value = activeReviewCounts.getOrDefault(extension.getId(), 0); + ratingCountStat.value = Optional.ofNullable(extension.getReviewCount()).orElse(0L); queryExt.statistics.add(ratingCountStat); } diff --git a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java index c898d05df..ef2525cfe 100644 --- a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java +++ b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java @@ -30,6 +30,7 @@ public class CacheService { public static final String CACHE_EXTENSION_JSON = "extension.json"; public static final String CACHE_LATEST_EXTENSION_VERSION = "latest.extension.version"; public static final String CACHE_NAMESPACE_DETAILS_JSON = "namespace.details.json"; + public static final String CACHE_AVERAGE_REVIEW_RATING = "average.review.rating"; public static final String GENERATOR_EXTENSION_JSON = "extensionJsonCacheKeyGenerator"; public static final String GENERATOR_LATEST_EXTENSION_VERSION = "latestExtensionVersionCacheKeyGenerator"; @@ -83,8 +84,10 @@ public void evictExtensionJsons(Extension extension) { var namespaceName = extension.getNamespace().getName(); var extensionName = extension.getName(); + var targetPlatforms = new ArrayList<>(TargetPlatform.TARGET_PLATFORM_NAMES); + targetPlatforms.add("null"); for(var version : versions) { - for(var targetPlatform : TargetPlatform.TARGET_PLATFORM_NAMES) { + for(var targetPlatform : targetPlatforms) { cache.evictIfPresent(extensionJsonCacheKey.generate(namespaceName, extensionName, targetPlatform, version)); } } diff --git a/server/src/main/java/org/eclipse/openvsx/cache/ExtensionJsonCacheKeyGenerator.java b/server/src/main/java/org/eclipse/openvsx/cache/ExtensionJsonCacheKeyGenerator.java index 05e6d5a05..82edb4c20 100644 --- a/server/src/main/java/org/eclipse/openvsx/cache/ExtensionJsonCacheKeyGenerator.java +++ b/server/src/main/java/org/eclipse/openvsx/cache/ExtensionJsonCacheKeyGenerator.java @@ -18,7 +18,8 @@ public class ExtensionJsonCacheKeyGenerator implements KeyGenerator { @Override public Object generate(Object target, Method method, Object... params) { - return generate((String) params[0], (String) params[1], (String) params[2], (String) params[3]); + var version = params.length == 4 ? (String) params[3] : "latest"; + return generate((String) params[0], (String) params[1], (String) params[2], version); } public String generate(String namespaceName, String extensionName, String targetPlatform, String version) { diff --git a/server/src/main/java/org/eclipse/openvsx/entities/Extension.java b/server/src/main/java/org/eclipse/openvsx/entities/Extension.java index 5a2789114..0ce37ea83 100644 --- a/server/src/main/java/org/eclipse/openvsx/entities/Extension.java +++ b/server/src/main/java/org/eclipse/openvsx/entities/Extension.java @@ -16,7 +16,6 @@ import java.util.stream.Collectors; import javax.persistence.*; -import javax.transaction.Transactional; import org.eclipse.openvsx.search.ExtensionSearch; @@ -46,6 +45,8 @@ public class Extension implements Serializable { Double averageRating; + Long reviewCount; + int downloadCount; LocalDateTime publishedDate; @@ -61,7 +62,6 @@ public ExtensionSearch toSearch(ExtensionVersion latest) { search.name = this.getName(); search.namespace = this.getNamespace().getName(); search.extensionId = search.namespace + "." + search.name; - search.averageRating = this.getAverageRating(); search.downloadCount = this.getDownloadCount(); search.targetPlatforms = this.getVersions().stream() .map(ExtensionVersion::getTargetPlatform) @@ -125,6 +125,14 @@ public void setAverageRating(Double averageRating) { this.averageRating = averageRating; } + public Long getReviewCount() { + return reviewCount; + } + + public void setReviewCount(Long reviewCount) { + this.reviewCount = reviewCount; + } + public int getDownloadCount() { return downloadCount; } @@ -170,12 +178,13 @@ public boolean equals(Object o) { && Objects.equals(namespace, extension.namespace) && Objects.equals(versions, extension.versions) && Objects.equals(averageRating, extension.averageRating) + && Objects.equals(reviewCount, extension.reviewCount) && Objects.equals(publishedDate, extension.publishedDate) && Objects.equals(lastUpdatedDate, extension.lastUpdatedDate); } @Override public int hashCode() { - return Objects.hash(id, publicId, name, namespace, versions, active, averageRating, downloadCount, publishedDate, lastUpdatedDate); + return Objects.hash(id, publicId, name, namespace, versions, active, averageRating, reviewCount, downloadCount, publishedDate, lastUpdatedDate); } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/entities/ExtensionVersion.java b/server/src/main/java/org/eclipse/openvsx/entities/ExtensionVersion.java index 0247261d3..90f09aa20 100644 --- a/server/src/main/java/org/eclipse/openvsx/entities/ExtensionVersion.java +++ b/server/src/main/java/org/eclipse/openvsx/entities/ExtensionVersion.java @@ -190,6 +190,7 @@ public SearchEntryJson toSearchEntryJson() { entry.name = extension.getName(); entry.namespace = extension.getNamespace().getName(); entry.averageRating = extension.getAverageRating(); + entry.reviewCount = extension.getReviewCount(); entry.downloadCount = extension.getDownloadCount(); entry.version = this.getVersion(); entry.timestamp = TimeUtil.toUTCString(this.getTimestamp()); diff --git a/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java b/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java index 33b6313dc..f8ff23603 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java @@ -61,6 +61,10 @@ public class SearchEntryJson implements Serializable { @Max(5) public Double averageRating; + @Schema(description = "Number of reviews") + @Min(0) + public Long reviewCount; + @Schema(description = "Number of downloads of the extension package") @Min(0) public int downloadCount; diff --git a/server/src/main/java/org/eclipse/openvsx/mirror/DataMirrorService.java b/server/src/main/java/org/eclipse/openvsx/mirror/DataMirrorService.java index 2591443ed..b95c8007a 100644 --- a/server/src/main/java/org/eclipse/openvsx/mirror/DataMirrorService.java +++ b/server/src/main/java/org/eclipse/openvsx/mirror/DataMirrorService.java @@ -208,6 +208,7 @@ public void updateMetadata(String namespaceName, String extensionName, Extension var extension = repositories.findExtension(extensionName, namespaceName); extension.setDownloadCount(latest.downloadCount); extension.setAverageRating(latest.averageRating); + extension.setReviewCount(latest.reviewCount); var remoteReviews = upstream.getReviews(namespaceName, extensionName); var localReviews = repositories.findAllReviews(extension) diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionJooqRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionJooqRepository.java index 3af51ae7e..635ca6fc7 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionJooqRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionJooqRepository.java @@ -20,8 +20,6 @@ import java.util.Collection; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import static org.eclipse.openvsx.jooq.Tables.*; @@ -53,24 +51,13 @@ var record = findAllActive() return record != null ? toExtension(record) : null; } - public Map findAllActiveReviewCountsById(Collection ids) { - var count = DSL.count(EXTENSION_REVIEW.ID).as("count"); - return dsl.select(EXTENSION_REVIEW.EXTENSION_ID, count) - .from(EXTENSION_REVIEW) - .where(EXTENSION_REVIEW.ACTIVE.eq(true)) - .and(EXTENSION_REVIEW.EXTENSION_ID.in(ids)) - .groupBy(EXTENSION_REVIEW.EXTENSION_ID) - .fetch() - .stream() - .collect(Collectors.toMap(r -> r.get(EXTENSION_REVIEW.EXTENSION_ID), r -> r.get(count))); - } - private SelectConditionStep findAllActive() { return dsl.select( EXTENSION.ID, EXTENSION.PUBLIC_ID, EXTENSION.NAME, EXTENSION.AVERAGE_RATING, + EXTENSION.REVIEW_COUNT, EXTENSION.DOWNLOAD_COUNT, EXTENSION.PUBLISHED_DATE, EXTENSION.LAST_UPDATED_DATE, @@ -93,6 +80,7 @@ private Extension toExtension(Record record) { extension.setPublicId(record.get(EXTENSION.PUBLIC_ID)); extension.setName(record.get(EXTENSION.NAME)); extension.setAverageRating(record.get(EXTENSION.AVERAGE_RATING)); + extension.setReviewCount(record.get(EXTENSION.REVIEW_COUNT)); extension.setDownloadCount(record.get(EXTENSION.DOWNLOAD_COUNT)); extension.setPublishedDate(record.get(EXTENSION.PUBLISHED_DATE)); extension.setLastUpdatedDate(record.get(EXTENSION.LAST_UPDATED_DATE)); diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionReviewRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionReviewRepository.java index 79d43bf32..d32b0f547 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionReviewRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionReviewRepository.java @@ -9,6 +9,7 @@ ********************************************************************************/ package org.eclipse.openvsx.repositories; +import org.springframework.cache.annotation.Cacheable; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.Repository; import org.springframework.data.util.Streamable; @@ -21,6 +22,8 @@ import java.time.LocalDateTime; +import static org.eclipse.openvsx.cache.CacheService.CACHE_AVERAGE_REVIEW_RATING; + public interface ExtensionReviewRepository extends Repository { Streamable findByExtension(Extension extension); @@ -30,4 +33,11 @@ public interface ExtensionReviewRepository extends Repository findByExtensionAndUserAndActiveTrue(Extension extension, UserData user); long countByExtensionAndActiveTrue(Extension extension); + + @Cacheable(CACHE_AVERAGE_REVIEW_RATING) + @Query("select coalesce(avg(r.rating),0) from ExtensionReview r where r.active = true") + double averageRatingAndActiveTrue(); + + @Query("select avg(r.rating) from ExtensionReview r where r.active = true and r.extension = ?1") + Double averageRatingAndActiveTrue(Extension extension); } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java index 3e92dad29..d6206fcc3 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/ExtensionVersionJooqRepository.java @@ -132,6 +132,7 @@ private SelectConditionStep findAllActive() { EXTENSION.PUBLIC_ID, EXTENSION.NAME, EXTENSION.AVERAGE_RATING, + EXTENSION.REVIEW_COUNT, EXTENSION.DOWNLOAD_COUNT, EXTENSION.PUBLISHED_DATE, EXTENSION.LAST_UPDATED_DATE, @@ -190,6 +191,7 @@ private ExtensionVersion toExtensionVersionFull(Record record) { var extension = extVersion.getExtension(); extension.setPublicId(record.get(EXTENSION.PUBLIC_ID)); extension.setAverageRating(record.get(EXTENSION.AVERAGE_RATING)); + extension.setReviewCount(record.get(EXTENSION.REVIEW_COUNT)); extension.setDownloadCount(record.get(EXTENSION.DOWNLOAD_COUNT)); extension.setPublishedDate(record.get(EXTENSION.PUBLISHED_DATE)); extension.setLastUpdatedDate(record.get(EXTENSION.LAST_UPDATED_DATE)); diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java index 1f324087f..ec03f7b58 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java @@ -318,10 +318,6 @@ public List findResourceFileResources(long extVersionId, String pr return fileResourceJooqRepo.findAllResources(extVersionId, prefix); } - public Map findActiveReviewCountsByExtensionId(Collection extensionIds) { - return extensionJooqRepo.findAllActiveReviewCountsById(extensionIds); - } - public List findNamespaceMemberships(Collection namespaceIds) { return namespaceMembershipJooqRepo.findAllByNamespaceId(namespaceIds); } @@ -413,4 +409,12 @@ public Streamable findNotMigratedVsixManifests() { private Streamable findNotMigratedItems(String migrationScript) { return migrationItemRepo.findByMigrationScriptAndMigrationScheduledFalseOrderById(migrationScript); } + + public double getAverageReviewRating() { + return extensionReviewRepo.averageRatingAndActiveTrue(); + } + + public Double getAverageReviewRating(Extension extension) { + return extensionReviewRepo.averageRatingAndActiveTrue(extension); + } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java b/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java index ac69f5125..0bdce690b 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java @@ -12,6 +12,7 @@ import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.eclipse.openvsx.util.TargetPlatform; import org.eclipse.openvsx.util.VersionService; @@ -27,6 +28,7 @@ import javax.transaction.Transactional; +import static org.eclipse.openvsx.cache.CacheService.CACHE_AVERAGE_REVIEW_RATING; import static org.eclipse.openvsx.cache.CacheService.CACHE_DATABASE_SEARCH; /** @@ -51,8 +53,9 @@ public boolean isEnabled() { @Autowired VersionService versions; - @Cacheable(CACHE_DATABASE_SEARCH) @Transactional + @Cacheable(CACHE_DATABASE_SEARCH) + @CacheEvict(value = CACHE_AVERAGE_REVIEW_RATING, allEntries = true) public SearchHits search(ISearchService.Options options) { // grab all extensions var matchingExtensions = repositories.findAllActiveExtensions(); @@ -95,37 +98,34 @@ public SearchHits search(ISearchService.Options options) { }); } - List sortedExtensions; - // need to perform the sortBy () - // 'relevance' | 'timestamp' | 'averageRating' | 'downloadCount'; + // 'relevance' | 'timestamp' | 'rating' | 'downloadCount'; - if ("relevance".equals(options.sortBy)) { - // for relevance we're using relevance service to get the relevance item + Stream searchEntries; + if("relevance".equals(options.sortBy) || "rating".equals(options.sortBy)) { var searchStats = new SearchStats(repositories); - - // needs to add relevance on extensions - sortedExtensions = matchingExtensions - .map(extension -> relevanceService.toSearchEntry(extension, searchStats)) - .stream() - .sorted(new RelevanceComparator()) - .collect(Collectors.toList()); + searchEntries = matchingExtensions.stream().map(extension -> relevanceService.toSearchEntry(extension, searchStats)); } else { - sortedExtensions = matchingExtensions.stream() - .map(extension -> { - var latest = versions.getLatest(extension, null, false, true); - return extension.toSearch(latest); - }) - .collect(Collectors.toList()); - if ("downloadCount".equals(options.sortBy)) { - sortedExtensions.sort(new DownloadedCountComparator()); - } else if ("averageRating".equals(options.sortBy)) { - sortedExtensions.sort(new AverageRatingComparator()); - } else if ("timestamp".equals(options.sortBy)) { - sortedExtensions.sort(new TimestampComparator()); - } + searchEntries = matchingExtensions.stream().map(extension -> { + var latest = versions.getLatest(extension, null, false, true); + return extension.toSearch(latest); + }); } + var comparators = new HashMap<>(Map.of( + "relevance", new RelevanceComparator(), + "timestamp", new TimestampComparator(), + "rating", new RatingComparator(), + "downloadCount", new DownloadedCountComparator() + )); + + var comparator = comparators.get(options.sortBy); + if(comparator != null) { + searchEntries = searchEntries.sorted(comparator); + } + + var sortedExtensions = searchEntries.collect(Collectors.toList()); + // need to do sortOrder // 'asc' | 'desc'; if ("desc".equals(options.sortOrder)) { @@ -193,17 +193,17 @@ public int compare(ExtensionSearch ext1, ExtensionSearch ext2) { /** * Sort by averageRating */ - class AverageRatingComparator implements Comparator { + class RatingComparator implements Comparator { @Override public int compare(ExtensionSearch ext1, ExtensionSearch ext2) { - if (ext1.averageRating == null) { + if (ext1.rating == null) { return -1; - } else if (ext2.averageRating == null) { + } else if (ext2.rating == null) { return 1; } - // averageRating - return Double.compare(ext1.averageRating, ext2.averageRating); + // rating + return Double.compare(ext1.rating, ext2.rating); } } diff --git a/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java b/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java index e07ab2dab..557e79e50 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java @@ -34,6 +34,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.cache.annotation.CacheEvict; import org.springframework.context.event.EventListener; import org.springframework.dao.DataAccessResourceFailureException; import org.springframework.data.domain.PageRequest; @@ -47,6 +48,8 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StopWatch; +import static org.eclipse.openvsx.cache.CacheService.CACHE_AVERAGE_REVIEW_RATING; + @Component public class ElasticSearchService implements ISearchService { @@ -91,6 +94,7 @@ public boolean isEnabled() { @EventListener @Transactional(readOnly = true) @Retryable(DataAccessResourceFailureException.class) + @CacheEvict(value = CACHE_AVERAGE_REVIEW_RATING, allEntries = true) public void initSearchIndex(ApplicationStartedEvent event) { if (!isEnabled() || !clearOnStart && searchOperations.indexOps(ExtensionSearch.class).exists()) { return; @@ -105,13 +109,14 @@ public void initSearchIndex(ApplicationStartedEvent event) { /** * Task scheduled once per day to soft-update the search index. This is necessary * because the relevance of index entries might consider the extension publishing - * timestamps in relation to the current time. + * timestamps in relation to the current time or the extension rating. */ @Scheduled(cron = "0 0 4 * * *", zone = "UTC") @Transactional(readOnly = true) @Retryable(DataAccessResourceFailureException.class) + @CacheEvict(value = CACHE_AVERAGE_REVIEW_RATING, allEntries = true) public void updateSearchIndex() { - if (!isEnabled() || Math.abs(timestampRelevance) < 0.01) { + if (!isEnabled()) { return; } var stopWatch = new StopWatch(); @@ -327,7 +332,7 @@ private void sortResults(NativeSearchQueryBuilder queryBuilder, String sortOrder var types = Map.of( "relevance", "float", - "averageRating", "float", + "rating", "float", "timestamp", "long", "downloadCount", "integer" ); diff --git a/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java b/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java index 154aa8eb7..bb5a3dbbf 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java +++ b/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java @@ -46,7 +46,7 @@ public class ExtensionSearch implements Serializable { @Nullable @Field(index = false, type = FieldType.Float) - public Double averageRating; + public Double rating; @Field(index = false) public int downloadCount; @@ -71,7 +71,7 @@ public boolean equals(Object o) { && Objects.equals(targetPlatforms, that.targetPlatforms) && Objects.equals(displayName, that.displayName) && Objects.equals(description, that.description) - && Objects.equals(averageRating, that.averageRating) + && Objects.equals(rating, that.rating) && Objects.equals(categories, that.categories) && Objects.equals(tags, that.tags); } @@ -80,7 +80,7 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash( id, relevance, name, namespace, extensionId, targetPlatforms, displayName, description, timestamp, - averageRating, downloadCount, categories, tags + rating, downloadCount, categories, tags ); } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/search/RelevanceService.java b/server/src/main/java/org/eclipse/openvsx/search/RelevanceService.java index 1f8cfa935..f538e5e8e 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/RelevanceService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/RelevanceService.java @@ -12,13 +12,12 @@ import java.time.Duration; import java.time.LocalDateTime; -import java.time.ZoneOffset; +import java.util.Optional; import javax.annotation.PostConstruct; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.eclipse.openvsx.ExtensionService; import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.NamespaceMembership; @@ -86,23 +85,38 @@ void init() { protected ExtensionSearch toSearchEntry(Extension extension, SearchStats stats) { var latest = versions.getLatest(extension, null, false, true); var entry = extension.toSearch(latest); + entry.rating = calculateRating(extension, stats); + entry.relevance = calculateRelevance(extension, latest, stats, entry); + return entry; + } + + private double calculateRating(Extension extension, SearchStats stats) { + // IMDB rating formula, source: https://stackoverflow.com/a/1411268 + var padding = 100; + var reviews = Optional.ofNullable(extension.getReviewCount()).orElse(0L); + var averageRating = Optional.ofNullable(extension.getAverageRating()).orElse(0.0); + // The amount of "smoothing" applied to the rating is based on reviews in relation to padding. + return (averageRating * reviews + stats.averageReviewRating * padding) / (reviews + padding); + } + + private double calculateRelevance(Extension extension, ExtensionVersion latest, SearchStats stats, ExtensionSearch entry) { var ratingValue = 0.0; - if (entry.averageRating != null) { - var reviewCount = repositories.countActiveReviews(extension); + if (extension.getAverageRating() != null) { + var reviewCount = extension.getReviewCount(); // Reduce the rating relevance if there are only few reviews var countRelevance = saturate(reviewCount, 0.25); - ratingValue = (entry.averageRating / 5.0) * countRelevance; + ratingValue = (extension.getAverageRating() / 5.0) * countRelevance; } var downloadsValue = entry.downloadCount / stats.downloadRef; var timestamp = latest.getTimestamp(); var timestampValue = Duration.between(stats.oldest, timestamp).toSeconds() / stats.timestampRef; - entry.relevance = ratingRelevance * limit(ratingValue) + downloadsRelevance * limit(downloadsValue) + var relevance = ratingRelevance * limit(ratingValue) + downloadsRelevance * limit(downloadsValue) + timestampRelevance * limit(timestampValue); // Reduce the relevance value of unverified extensions if (!isVerified(latest)) { - entry.relevance *= unverifiedRelevance; + relevance *= unverifiedRelevance; } if (Double.isNaN(entry.relevance) || Double.isInfinite(entry.relevance)) { @@ -113,9 +127,10 @@ protected ExtensionSearch toSearchEntry(Extension extension, SearchStats stats) // Ignore exception } logger.error(message); - entry.relevance = 0.0; + relevance = 0.0; } - return entry; + + return relevance; } private double limit(double value) { @@ -144,6 +159,7 @@ public static class SearchStats { protected final double downloadRef; protected final double timestampRef; protected final LocalDateTime oldest; + protected final double averageReviewRating; public SearchStats(RepositoryService repositories) { var now = TimeUtil.getCurrentUTC(); @@ -152,6 +168,7 @@ public SearchStats(RepositoryService repositories) { this.downloadRef = maxDownloads * 1.5 + 100; this.oldest = oldestTimestamp == null ? now : oldestTimestamp; this.timestampRef = Duration.between(this.oldest, now).toSeconds() + 60; + this.averageReviewRating = repositories.getAverageReviewRating(); } } } diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountProcessor.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountProcessor.java index 091748dfa..04da8d002 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountProcessor.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountProcessor.java @@ -82,10 +82,11 @@ public List increaseDownloadCounts(Map> extensio } @Transactional //needs transaction for lazy-loading versions - public void evictExtensionJsons(List extensions) { + public void evictCaches(List extensions) { extensions.forEach(extension -> { extension = entityManager.merge(extension); cache.evictExtensionJsons(extension); + cache.evictLatestExtensionVersion(extension); }); } diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java index 328c0e829..918358408 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java @@ -127,7 +127,7 @@ public void updateDownloadCounts() { if(!files.isEmpty()) { var extensionDownloads = processor.processDownloadCounts(files); var updatedExtensions = processor.increaseDownloadCounts(extensionDownloads); - processor.evictExtensionJsons(updatedExtensions); + processor.evictCaches(updatedExtensions); processor.updateSearchEntries(updatedExtensions); } diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java index 718741dc0..c55faa326 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java @@ -247,7 +247,7 @@ public Map> getFileUrls(Collection e return type2Url; } - @Transactional(Transactional.TxType.MANDATORY) + @Transactional public void increaseDownloadCount(FileResource resource) { if(azureDownloadCountService.isEnabled()) { // don't count downloads twice diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/Extension.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/Extension.java index 1b1a6b0a5..a38fd46b2 100644 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/Extension.java +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/Extension.java @@ -17,7 +17,7 @@ import org.jooq.Index; import org.jooq.Name; import org.jooq.Record; -import org.jooq.Row9; +import org.jooq.Row10; import org.jooq.Schema; import org.jooq.Table; import org.jooq.TableField; @@ -94,6 +94,11 @@ public Class getRecordType() { */ public final TableField LAST_UPDATED_DATE = createField(DSL.name("last_updated_date"), SQLDataType.LOCALDATETIME(6), this, ""); + /** + * The column public.extension.review_count. + */ + public final TableField REVIEW_COUNT = createField(DSL.name("review_count"), SQLDataType.BIGINT, this, ""); + private Extension(Name alias, Table aliased) { this(alias, aliased, null); } @@ -188,11 +193,11 @@ public Extension rename(Name name) { } // ------------------------------------------------------------------------- - // Row9 type methods + // Row10 type methods // ------------------------------------------------------------------------- @Override - public Row9 fieldsRow() { - return (Row9) super.fieldsRow(); + public Row10 fieldsRow() { + return (Row10) super.fieldsRow(); } } diff --git a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/ExtensionRecord.java b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/ExtensionRecord.java index 13925589c..2ba41e704 100644 --- a/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/ExtensionRecord.java +++ b/server/src/main/jooq-gen/org/eclipse/openvsx/jooq/tables/records/ExtensionRecord.java @@ -9,8 +9,8 @@ import org.eclipse.openvsx.jooq.tables.Extension; import org.jooq.Field; import org.jooq.Record1; -import org.jooq.Record9; -import org.jooq.Row9; +import org.jooq.Record10; +import org.jooq.Row10; import org.jooq.impl.UpdatableRecordImpl; @@ -18,7 +18,7 @@ * This class is generated by jOOQ. */ @SuppressWarnings({ "all", "unchecked", "rawtypes" }) -public class ExtensionRecord extends UpdatableRecordImpl implements Record9 { +public class ExtensionRecord extends UpdatableRecordImpl implements Record10 { private static final long serialVersionUID = 1L; @@ -148,6 +148,20 @@ public LocalDateTime getLastUpdatedDate() { return (LocalDateTime) get(8); } + /** + * Setter for public.extension.review_count. + */ + public void setReviewCount(Long value) { + set(9, value); + } + + /** + * Getter for public.extension.review_count. + */ + public Long getReviewCount() { + return (Long) get(9); + } + // ------------------------------------------------------------------------- // Primary key information // ------------------------------------------------------------------------- @@ -158,17 +172,17 @@ public Record1 key() { } // ------------------------------------------------------------------------- - // Record9 type implementation + // Record10 type implementation // ------------------------------------------------------------------------- @Override - public Row9 fieldsRow() { - return (Row9) super.fieldsRow(); + public Row10 fieldsRow() { + return (Row10) super.fieldsRow(); } @Override - public Row9 valuesRow() { - return (Row9) super.valuesRow(); + public Row10 valuesRow() { + return (Row10) super.valuesRow(); } @Override @@ -216,6 +230,11 @@ public Field field9() { return Extension.EXTENSION.LAST_UPDATED_DATE; } + @Override + public Field field10() { + return Extension.EXTENSION.REVIEW_COUNT; + } + @Override public Long component1() { return getId(); @@ -261,6 +280,11 @@ public LocalDateTime component9() { return getLastUpdatedDate(); } + @Override + public Long component10() { + return getReviewCount(); + } + @Override public Long value1() { return getId(); @@ -306,6 +330,11 @@ public LocalDateTime value9() { return getLastUpdatedDate(); } + @Override + public Long value10() { + return getReviewCount(); + } + @Override public ExtensionRecord value1(Long value) { setId(value); @@ -361,7 +390,13 @@ public ExtensionRecord value9(LocalDateTime value) { } @Override - public ExtensionRecord values(Long value1, Double value2, Integer value3, String value4, Long value5, String value6, Boolean value7, LocalDateTime value8, LocalDateTime value9) { + public ExtensionRecord value10(Long value) { + setReviewCount(value); + return this; + } + + @Override + public ExtensionRecord values(Long value1, Double value2, Integer value3, String value4, Long value5, String value6, Boolean value7, LocalDateTime value8, LocalDateTime value9, Long value10) { value1(value1); value2(value2); value3(value3); @@ -371,6 +406,7 @@ public ExtensionRecord values(Long value1, Double value2, Integer value3, String value7(value7); value8(value8); value9(value9); + value10(value10); return this; } @@ -388,7 +424,7 @@ public ExtensionRecord() { /** * Create a detached, initialised ExtensionRecord */ - public ExtensionRecord(Long id, Double averageRating, Integer downloadCount, String name, Long namespaceId, String publicId, Boolean active, LocalDateTime publishedDate, LocalDateTime lastUpdatedDate) { + public ExtensionRecord(Long id, Double averageRating, Integer downloadCount, String name, Long namespaceId, String publicId, Boolean active, LocalDateTime publishedDate, LocalDateTime lastUpdatedDate, Long reviewCount) { super(Extension.EXTENSION); setId(id); @@ -400,5 +436,6 @@ public ExtensionRecord(Long id, Double averageRating, Integer downloadCount, Str setActive(active); setPublishedDate(publishedDate); setLastUpdatedDate(lastUpdatedDate); + setReviewCount(reviewCount); } } diff --git a/server/src/main/resources/db/migration/V1_33__Extension_ReviewCount.sql b/server/src/main/resources/db/migration/V1_33__Extension_ReviewCount.sql new file mode 100644 index 000000000..86f94a4f2 --- /dev/null +++ b/server/src/main/resources/db/migration/V1_33__Extension_ReviewCount.sql @@ -0,0 +1,11 @@ +ALTER TABLE extension ADD COLUMN review_count BIGINT; + +UPDATE extension e +SET review_count = reviews +FROM ( + SELECT extension_id, COUNT(id) reviews + FROM extension_review + WHERE active = TRUE + GROUP BY extension_id +) r +WHERE e.id = r.extension_id; \ No newline at end of file diff --git a/server/src/main/resources/ehcache.xml b/server/src/main/resources/ehcache.xml index 740de09c1..3a37f5f9d 100644 --- a/server/src/main/resources/ehcache.xml +++ b/server/src/main/resources/ehcache.xml @@ -2,17 +2,17 @@ xmlns="http://www.ehcache.org/v3" xsi:schemaLocation="http://www.ehcache.org/v3 http://www.ehcache.org/schema/ehcache-core.xsd"> - + - 3600 + - 1024 - 32 - 128 + 1 + 1 + 2 - + 3600 @@ -22,17 +22,15 @@ 128 - + 3600 1024 - 32 - 128 - + 3600 @@ -42,7 +40,7 @@ 128 - + 3600 diff --git a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java index 705a3e795..79a735bf6 100644 --- a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java @@ -1741,8 +1741,6 @@ private void mockExtensionVersions(String targetPlatform, List values, B Mockito.when(repositories.findActiveExtensionVersions(Set.of(extension.getId()), null)) .thenReturn(versions); - Mockito.when(repositories.findActiveReviewCountsByExtensionId(Set.of(extension.getId()))) - .thenReturn(Collections.emptyMap()); var fileTypes = List.of(DOWNLOAD, MANIFEST, ICON, README, LICENSE, CHANGELOG); Mockito.when(repositories.findFileResourcesByExtensionVersionIdAndType(List.of(3L), fileTypes)) .thenReturn(Collections.emptyList()); @@ -1783,8 +1781,6 @@ private ExtensionVersion mockExtensionVersion() { Mockito.when(repositories.findActiveExtensionVersions(Set.of(extension.getId()), null)) .thenReturn(List.of(extVersion)); - Mockito.when(repositories.findActiveReviewCountsByExtensionId(Set.of(extension.getId()))) - .thenReturn(Collections.emptyMap()); var fileTypes = List.of(DOWNLOAD, MANIFEST, ICON, README, LICENSE, CHANGELOG); Mockito.when(repositories.findFileResourcesByExtensionVersionIdAndType(Set.of(extVersion.getId()), fileTypes)) .thenReturn(Collections.emptyList()); diff --git a/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java b/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java index 32a2295b6..56c653f1d 100644 --- a/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java @@ -658,8 +658,6 @@ private Extension mockSearch(String targetPlatform, String namespaceName, boolea .thenReturn(results); var ids = List.of(extension.getId()); - Mockito.when(repositories.findActiveReviewCountsByExtensionId(ids)) - .thenReturn(Map.of(extension.getId(), 10)); Mockito.when(repositories.findActiveExtension(extension.getName(), extension.getNamespace().getName())) .thenReturn(extension); @@ -678,6 +676,7 @@ private Extension mockExtension() { extension.setPublicId("test-1"); extension.setName("vscode-yaml"); extension.setAverageRating(3.0); + extension.setReviewCount(10L); extension.setDownloadCount(100); extension.setPublishedDate(LocalDateTime.parse("1999-12-01T09:00")); extension.setLastUpdatedDate(LocalDateTime.parse("2000-01-01T10:00")); @@ -800,8 +799,6 @@ private ExtensionVersion mockExtensionVersion(String targetPlatform) throws Json .thenReturn(Streamable.of(extVersion)); Mockito.when(repositories.countMemberships(namespace, NamespaceMembership.ROLE_OWNER)) .thenReturn(0L); - Mockito.when(repositories.countActiveReviews(extension)) - .thenReturn(10L); var extensionFile = new FileResource(); extensionFile.setExtension(extVersion); extensionFile.setName("redhat.vscode-yaml-0.5.2.vsix"); diff --git a/server/src/test/java/org/eclipse/openvsx/cache/CacheServiceTest.java b/server/src/test/java/org/eclipse/openvsx/cache/CacheServiceTest.java index e495682f1..ff9f36fc6 100644 --- a/server/src/test/java/org/eclipse/openvsx/cache/CacheServiceTest.java +++ b/server/src/test/java/org/eclipse/openvsx/cache/CacheServiceTest.java @@ -16,8 +16,10 @@ import org.eclipse.openvsx.entities.*; import org.eclipse.openvsx.json.ExtensionJson; import org.eclipse.openvsx.json.ReviewJson; +import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.security.IdPrincipal; import org.eclipse.openvsx.util.TimeUtil; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -43,8 +45,7 @@ import static org.eclipse.openvsx.cache.CacheService.CACHE_EXTENSION_JSON; import static org.eclipse.openvsx.entities.FileResource.DOWNLOAD; import static org.eclipse.openvsx.entities.FileResource.STORAGE_DB; -import static org.junit.Assert.*; -import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.*; @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles("test") @@ -68,6 +69,9 @@ public class CacheServiceTest { @Autowired EntityManager entityManager; + @Autowired + RepositoryService repositories; + @Test @Transactional public void testGetExtension() { @@ -272,6 +276,31 @@ public void testUpdateExtension() { assertEquals(json, cachedJson); } + @Test + @Transactional + public void testAverageReviewRating() { + var user = insertAdmin(); + var extVersion = insertExtensionVersion(); + // no reviews in database + assertEquals(0L, repositories.getAverageReviewRating()); + + var review = new ExtensionReview(); + review.setRating(3); + review.setActive(true); + review.setExtension(extVersion.getExtension()); + review.setTimestamp(LocalDateTime.now()); + review.setUser(user); + entityManager.persist(review); + + // returns cached value + assertEquals(0L, repositories.getAverageReviewRating()); + + cache.getCache(CacheService.CACHE_AVERAGE_REVIEW_RATING).clear(); + + // returns new value from database + assertEquals(3L, repositories.getAverageReviewRating()); + } + private void setLoggedInUser(UserData user) { var principal = new IdPrincipal(user.getId(), user.getLoginName(), List.of((GrantedAuthority) () -> "github")); var authentication = new TestingAuthenticationToken(principal, null); diff --git a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java index 031738e9c..52cdfffd1 100644 --- a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java +++ b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java @@ -151,7 +151,6 @@ void testExecuteQueries() { () -> repositories.topNamespaceExtensions(NOW, 1), () -> repositories.topNamespaceExtensionVersions(NOW, 1), () -> repositories.findFileResourcesByExtensionVersionIdAndType(LONG_LIST, STRING_LIST), - () -> repositories.findActiveReviewCountsByExtensionId(LONG_LIST), () -> repositories.findActiveExtensionVersionsByVersion("version", "extensionName", "namespaceName"), () -> repositories.findResourceFileResources(1L, "prefix"), () -> repositories.findActiveExtensionVersions(LONG_LIST, "targetPlatform"), @@ -164,7 +163,9 @@ void testExecuteQueries() { () -> repositories.findActiveExtensionVersionsByExtensionName("targetPlatform", "extensionName", "namespaceName"), () -> repositories.findActiveExtensionVersionsByExtensionName("targetPlatform", "extensionName"), () -> repositories.findActiveExtensionVersionsByNamespacePublicId("targetPlatform", "namespacePublicId"), - () -> repositories.findAllNotMatchingByExtensionId(STRING_LIST) + () -> repositories.findAllNotMatchingByExtensionId(STRING_LIST), + () -> repositories.getAverageReviewRating(null), + () -> repositories.getAverageReviewRating() ); // check that we did not miss anything diff --git a/server/src/test/java/org/eclipse/openvsx/search/DatabaseSearchServiceTest.java b/server/src/test/java/org/eclipse/openvsx/search/DatabaseSearchServiceTest.java index 6901776cb..ecd52432a 100644 --- a/server/src/test/java/org/eclipse/openvsx/search/DatabaseSearchServiceTest.java +++ b/server/src/test/java/org/eclipse/openvsx/search/DatabaseSearchServiceTest.java @@ -276,14 +276,14 @@ public void testSortByDownloadCount() throws Exception { } @Test - public void testSortByAverageRating() throws Exception { - var ext1 = mockExtension("yaml", 4.0, 0, 0, "redhat", Arrays.asList("Snippets", "Programming Languages")); - var ext2 = mockExtension("java", 5.0, 0, 0, "redhat", Arrays.asList("Snippets", "Programming Languages")); - var ext3 = mockExtension("openshift", 2.0, 0, 0, "redhat", Arrays.asList("Snippets", "Other")); - var ext4 = mockExtension("foo", 1.0, 0, 0, "bar", Arrays.asList("Other")); + public void testSortByRating() throws Exception { + var ext1 = mockExtension("yaml", 4.0, 1, 0, "redhat", Arrays.asList("Snippets", "Programming Languages")); + var ext2 = mockExtension("java", 5.0, 1, 0, "redhat", Arrays.asList("Snippets", "Programming Languages")); + var ext3 = mockExtension("openshift", 2.0, 1, 0, "redhat", Arrays.asList("Snippets", "Other")); + var ext4 = mockExtension("foo", 1.0, 1, 0, "bar", Arrays.asList("Other")); Mockito.when(repositories.findAllActiveExtensions()).thenReturn(Streamable.of(List.of(ext1, ext2, ext3, ext4))); - var searchOptions = new ISearchService.Options(null, null, TargetPlatform.NAME_UNIVERSAL, 50, 0, null, "averageRating", false); + var searchOptions = new ISearchService.Options(null, null, TargetPlatform.NAME_UNIVERSAL, 50, 0, null, "rating", false); var result = search.search(searchOptions); // all extensions should be there assertThat(result.getTotalHits()).isEqualTo(4); @@ -306,16 +306,16 @@ long getIdFromExtensionName(String extensionName) { return extensionName.hashCode(); } - private Extension mockExtension(String name, double averageRating, int ratingCount, int downloadCount, + private Extension mockExtension(String name, double averageRating, long ratingCount, int downloadCount, String namespaceName, List categories) { var extension = new Extension(); extension.setName(name); extension.setId(name.hashCode()); extension.setAverageRating(averageRating); + extension.setReviewCount(ratingCount); extension.setDownloadCount(downloadCount); extension.setActive(true); Mockito.when(entityManager.merge(extension)).thenReturn(extension); - Mockito.when(repositories.countActiveReviews(extension)).thenReturn((long) ratingCount); var namespace = new Namespace(); namespace.setName(namespaceName); extension.setNamespace(namespace); diff --git a/server/src/test/java/org/eclipse/openvsx/search/ElasticSearchServiceTest.java b/server/src/test/java/org/eclipse/openvsx/search/ElasticSearchServiceTest.java index 15c5e9f9b..2d376cc60 100644 --- a/server/src/test/java/org/eclipse/openvsx/search/ElasticSearchServiceTest.java +++ b/server/src/test/java/org/eclipse/openvsx/search/ElasticSearchServiceTest.java @@ -232,16 +232,16 @@ private MockIndex mockIndex(boolean exists) { return index; } - private Extension mockExtension(String name, String namespaceName, String userName, double averageRating, int ratingCount, int downloadCount, + private Extension mockExtension(String name, String namespaceName, String userName, double averageRating, long ratingCount, int downloadCount, LocalDateTime timestamp, boolean isUnverified, boolean isUnrelated) { var extension = new Extension(); extension.setName(name); extension.setId(name.hashCode()); extension.setAverageRating(averageRating); + extension.setReviewCount(ratingCount); extension.setDownloadCount(downloadCount); Mockito.when(entityManager.merge(extension)).thenReturn(extension); - Mockito.when(repositories.countActiveReviews(extension)) - .thenReturn((long) ratingCount); + var namespace = new Namespace(); namespace.setName(namespaceName); extension.setNamespace(namespace); diff --git a/webui/src/extension-registry-types.ts b/webui/src/extension-registry-types.ts index c7e72bdc1..2ae338b6b 100644 --- a/webui/src/extension-registry-types.ts +++ b/webui/src/extension-registry-types.ts @@ -53,6 +53,7 @@ export interface SearchEntry { engines?: { [engine: string]: string }; }[]; averageRating?: number; + reviewCount?: number; downloadCount?: number; displayName?: string; description?: string; @@ -235,5 +236,5 @@ export interface TargetPlatformVersion { } export type MembershipRole = 'contributor' | 'owner'; -export type SortBy = 'relevance' | 'timestamp' | 'averageRating' | 'downloadCount'; +export type SortBy = 'relevance' | 'timestamp' | 'rating' | 'downloadCount'; export type SortOrder = 'asc' | 'desc'; diff --git a/webui/src/pages/extension-list/extension-list-header.tsx b/webui/src/pages/extension-list/extension-list-header.tsx index 9f044ecca..6f79ea141 100644 --- a/webui/src/pages/extension-list/extension-list-header.tsx +++ b/webui/src/pages/extension-list/extension-list-header.tsx @@ -211,7 +211,7 @@ class ExtensionListHeaderComp extends React.ComponentRelevance Date Downloads - Rating + Rating @@ -114,6 +116,7 @@ class ExtensionListItemComponent extends React.Component + ({reviewCountFormatted}) From 2c73daf03adb25c0fc4ee502af1b54a5dbb55b3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Tron=C3=AD=C4=8Dek?= Date: Mon, 13 Feb 2023 13:39:33 +0100 Subject: [PATCH 08/45] Access token dialog improvements (#661) * Access token dialog improvements * Set focus on the Copy button --------- Co-authored-by: amvanbaren --- .../src/pages/user/generate-token-dialog.tsx | 21 +++++++++++-------- webui/src/pages/user/user-settings-tokens.tsx | 6 ++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/webui/src/pages/user/generate-token-dialog.tsx b/webui/src/pages/user/generate-token-dialog.tsx index bd473f912..14f66c379 100644 --- a/webui/src/pages/user/generate-token-dialog.tsx +++ b/webui/src/pages/user/generate-token-dialog.tsx @@ -45,17 +45,18 @@ class GenerateTokenDialogComponent extends React.Component { - this.setState({ open: true, posted: false }); - }; - - protected handleCancel = () => { - this.setState({ - open: false, + this.setState({ + open: true, + posted: false, description: '', token: undefined }); }; + protected handleClose = () => { + this.setState({ open: false }); + }; + protected handleDescriptionChange = (event: React.ChangeEvent) => { const description = event.target.value; let descriptionError: string | undefined; @@ -83,7 +84,7 @@ class GenerateTokenDialogComponent extends React.Component { - if (e.code === 'Enter') { + if (e.code === 'Enter' && this.state.open && !this.state.token) { this.handleGenerate(); } }; @@ -100,7 +101,7 @@ class GenerateTokenDialogComponent extends React.Component - + Generate new token @@ -140,10 +141,12 @@ class GenerateTokenDialogComponent extends React.Component {({ copy }) => ( { diff --git a/webui/src/pages/user/user-settings-tokens.tsx b/webui/src/pages/user/user-settings-tokens.tsx index 6384d79b2..a94fc1e3c 100644 --- a/webui/src/pages/user/user-settings-tokens.tsx +++ b/webui/src/pages/user/user-settings-tokens.tsx @@ -144,7 +144,8 @@ class UserSettingsTokensComponent extends React.Component + classes={{ root: this.props.classes.deleteBtn }} + disabled={this.state.loading}> Delete all @@ -178,7 +179,8 @@ class UserSettingsTokensComponent extends React.Component this.handleDelete(token)} - classes={{ root: this.props.classes.deleteBtn }}> + classes={{ root: this.props.classes.deleteBtn }} + disabled={this.state.loading}> Delete From 7bd25b658532aa272047bd20992cbc34538b4fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Tron=C3=AD=C4=8Dek?= Date: Tue, 14 Feb 2023 10:17:06 +0100 Subject: [PATCH 09/45] Search bar improvements (#662) * Search bar improvements * Remove 'Space' key in extension-list-header.tsx Change charCode 13 to key 'Enter' in namespace-input.tsx --------- Co-authored-by: amvanbaren --- webui/src/pages/admin-dashboard/namespace-input.tsx | 2 +- webui/src/pages/extension-list/extension-list-header.tsx | 7 +++++++ .../pages/extension-list/extension-list-searchfield.tsx | 4 +++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/webui/src/pages/admin-dashboard/namespace-input.tsx b/webui/src/pages/admin-dashboard/namespace-input.tsx index 740519740..e5d0aa9af 100644 --- a/webui/src/pages/admin-dashboard/namespace-input.tsx +++ b/webui/src/pages/admin-dashboard/namespace-input.tsx @@ -71,7 +71,7 @@ export const StyledInput: FunctionComponent = props => { placeholder={props.placeholder} onChange={onChangeInputValue} onKeyPress={(e: React.KeyboardEvent) => { - if (e.charCode === 13 && props.onSubmit) { + if (e.key === 'Enter' && props.onSubmit) { props.onSubmit(inputValue); } }} diff --git a/webui/src/pages/extension-list/extension-list-header.tsx b/webui/src/pages/extension-list/extension-list-header.tsx index 6f79ea141..3e9c98f6f 100644 --- a/webui/src/pages/extension-list/extension-list-header.tsx +++ b/webui/src/pages/extension-list/extension-list-header.tsx @@ -217,6 +217,13 @@ class ExtensionListHeaderComp extends React.Component { + if (e.key === 'Enter') { + e.preventDefault(); + this.handleSortOrderChange(); + } + }} onClick={this.handleSortOrderChange}> { this.state.sortOrder === 'asc' ? diff --git a/webui/src/pages/extension-list/extension-list-searchfield.tsx b/webui/src/pages/extension-list/extension-list-searchfield.tsx index 81ebc5401..d1573e567 100644 --- a/webui/src/pages/extension-list/extension-list-searchfield.tsx +++ b/webui/src/pages/extension-list/extension-list-searchfield.tsx @@ -72,8 +72,10 @@ export const ExtensionListSearchfield: FunctionComponent { - if (e.charCode === 13 && props.onSearchSubmit) { + if (e.key === 'Enter' && props.onSearchSubmit) { props.onSearchSubmit(props.searchQuery || ''); } }} From 30dbfefe1b5cfba10cbeb0bdf0619dd57a9e97ff Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Thu, 5 Jan 2023 10:44:01 +0200 Subject: [PATCH 10/45] [Admin] Add Ability to Change Namespace Fixes #427 Change Namespace Evict caches Update search entries --- .../java/org/eclipse/openvsx/AdminAPI.java | 15 + .../org/eclipse/openvsx/AdminService.java | 56 +++- .../openvsx/json/ChangeNamespaceJson.java | 23 ++ .../org/eclipse/openvsx/AdminAPITest.java | 269 ++++++++++++++++++ webui/src/extension-registry-service.ts | 18 ++ .../pages/admin-dashboard/namespace-admin.tsx | 1 + .../namespace-change-dialog.tsx | 120 ++++++++ .../pages/user/user-namespace-member-list.tsx | 2 +- .../user/user-settings-namespace-detail.tsx | 58 +++- 9 files changed, 552 insertions(+), 10 deletions(-) create mode 100644 server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java create mode 100644 webui/src/pages/admin-dashboard/namespace-change-dialog.tsx diff --git a/server/src/main/java/org/eclipse/openvsx/AdminAPI.java b/server/src/main/java/org/eclipse/openvsx/AdminAPI.java index ae991dc7f..b1089ad0e 100644 --- a/server/src/main/java/org/eclipse/openvsx/AdminAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/AdminAPI.java @@ -273,6 +273,21 @@ public ResponseEntity createNamespace(@RequestBody NamespaceJson nam } } + @PostMapping( + path = "/admin/change-namespace", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE + ) + public ResponseEntity changeNamespace(@RequestBody ChangeNamespaceJson json) { + try { + admins.checkAdminUser(); + admins.changeNamespace(json); + return ResponseEntity.ok(ResultJson.success("Changed namespace " + json.oldNamespace + " to " + json.newNamespace)); + } catch (ErrorResultException exc) { + return exc.toResponseEntity(); + } + } + @GetMapping( path = "/admin/namespace/{namespaceName}/members", produces = MediaType.APPLICATION_JSON_VALUE diff --git a/server/src/main/java/org/eclipse/openvsx/AdminService.java b/server/src/main/java/org/eclipse/openvsx/AdminService.java index ed78cef3e..db1910b9e 100644 --- a/server/src/main/java/org/eclipse/openvsx/AdminService.java +++ b/server/src/main/java/org/eclipse/openvsx/AdminService.java @@ -183,10 +183,7 @@ public ResultJson editNamespaceMember(String namespaceName, String userName, Str @Transactional(rollbackOn = ErrorResultException.class) public ResultJson createNamespace(NamespaceJson json) { - var namespaceIssue = validator.validateNamespace(json.name); - if (namespaceIssue.isPresent()) { - throw new ErrorResultException(namespaceIssue.get().toString()); - } + validateNamespace(json.name); var namespace = repositories.findNamespace(json.name); if (namespace != null) { throw new ErrorResultException("Namespace already exists: " + namespace.getName()); @@ -197,6 +194,57 @@ public ResultJson createNamespace(NamespaceJson json) { return ResultJson.success("Created namespace " + namespace.getName()); } + @Transactional + public void changeNamespace(ChangeNamespaceJson json) { + if(Strings.isNullOrEmpty(json.oldNamespace)) { + throw new ErrorResultException("Old namespace must have a value"); + } + if(Strings.isNullOrEmpty(json.newNamespace)) { + throw new ErrorResultException("New namespace must have a value"); + } + + var oldNamespace = repositories.findNamespace(json.oldNamespace); + if (oldNamespace == null) { + throw new ErrorResultException("Old namespace doesn't exists: " + json.oldNamespace); + } + + var newNamespace = repositories.findNamespace(json.newNamespace); + if(newNamespace != null && !json.mergeIfNewNamespaceAlreadyExists) { + throw new ErrorResultException("New namespace already exists: " + json.newNamespace); + } + if(newNamespace == null) { + validateNamespace(json.newNamespace); + newNamespace = new Namespace(); + newNamespace.setName(json.newNamespace); + entityManager.persist(newNamespace); + } + + var extensions = repositories.findExtensions(oldNamespace); + for(var extension : extensions) { + cache.evictExtensionJsons(extension); + cache.evictLatestExtensionVersion(extension); + extension.setNamespace(newNamespace); + } + + var memberships = repositories.findMemberships(oldNamespace); + for(var membership : memberships) { + membership.setNamespace(newNamespace); + } + + if(json.removeOldNamespace) { + entityManager.remove(oldNamespace); + } + + search.updateSearchEntries(extensions.toList()); + } + + private void validateNamespace(String namespace) { + var namespaceIssue = validator.validateNamespace(namespace); + if (namespaceIssue.isPresent()) { + throw new ErrorResultException(namespaceIssue.get().toString()); + } + } + public UserPublishInfoJson getUserPublishInfo(String provider, String loginName) { var user = repositories.findUserByLoginName(provider, loginName); if (user == null) { diff --git a/server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java b/server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java new file mode 100644 index 000000000..20cd41469 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java @@ -0,0 +1,23 @@ +/** ****************************************************************************** + * Copyright (c) 2022 Precies. Software and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.json; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * Used to change a namespace + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ChangeNamespaceJson { + public String oldNamespace; + public String newNamespace; + public boolean removeOldNamespace; + public boolean mergeIfNewNamespaceAlreadyExists; +} diff --git a/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java b/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java index e11c793a6..5b8b6eda7 100644 --- a/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java @@ -1023,6 +1023,275 @@ public void testAdminOnTheFlyReportJson() throws Exception { }))); } + @Test + public void testChangeNamespace() throws Exception { + mockAdminUser(); + var foo = new Namespace(); + foo.setName("foo"); + Mockito.when(repositories.findNamespace(foo.getName())).thenReturn(foo); + + var bar = new Namespace(); + bar.setName("bar"); + Mockito.when(repositories.findNamespace(bar.getName())).thenReturn(null); + + var extension = Mockito.mock(Extension.class); + Mockito.when(repositories.findExtensions(foo)).thenReturn(Streamable.of(extension)); + + var membership = Mockito.mock(NamespaceMembership.class); + Mockito.when(repositories.findMemberships(foo)).thenReturn(Streamable.of(membership)); + + var content = "{" + + "\"oldNamespace\": \"foo\", " + + "\"newNamespace\": \"bar\", " + + "\"removeOldNamespace\": false, " + + "\"mergeIfNewNamespaceAlreadyExists\": true" + + "}"; + + mockMvc.perform(post("/admin/change-namespace") + .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) + .with(csrf().asHeader()) + .content(content) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().json(successJson("Changed namespace foo to bar"))) + .andExpect(result -> Mockito.verify(extension).setNamespace(bar)) + .andExpect(result -> Mockito.verify(membership).setNamespace(bar)); + } + + @Test + public void testChangeNamespaceOldNamespaceNull() throws Exception { + mockAdminUser(); + var content = "{" + + "\"oldNamespace\": null, " + + "\"newNamespace\": \"bar\", " + + "\"removeOldNamespace\": false, " + + "\"mergeIfNewNamespaceAlreadyExists\": true" + + "}"; + + mockMvc.perform(post("/admin/change-namespace") + .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) + .with(csrf().asHeader()) + .content(content) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(content().json(errorJson("Old namespace must have a value"))); + } + + @Test + public void testChangeNamespaceOldNamespaceEmpty() throws Exception { + mockAdminUser(); + var content = "{" + + "\"oldNamespace\": \"\", " + + "\"newNamespace\": \"bar\", " + + "\"removeOldNamespace\": false, " + + "\"mergeIfNewNamespaceAlreadyExists\": true" + + "}"; + + mockMvc.perform(post("/admin/change-namespace") + .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) + .with(csrf().asHeader()) + .content(content) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(content().json(errorJson("Old namespace must have a value"))); + } + + @Test + public void testChangeNamespaceOldNamespaceDoesNotExist() throws Exception { + mockAdminUser(); + Mockito.when(repositories.findNamespace("foo")).thenReturn(null); + + var bar = new Namespace(); + bar.setName("bar"); + Mockito.when(repositories.findNamespace(bar.getName())).thenReturn(bar); + + var content = "{" + + "\"oldNamespace\": \"foo\", " + + "\"newNamespace\": \"bar\", " + + "\"removeOldNamespace\": false, " + + "\"mergeIfNewNamespaceAlreadyExists\": true" + + "}"; + + mockMvc.perform(post("/admin/change-namespace") + .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) + .with(csrf().asHeader()) + .content(content) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(content().json(errorJson("Old namespace doesn't exists: foo"))); + } + + @Test + public void testChangeNamespaceNewNamespaceNull() throws Exception { + mockAdminUser(); + var content = "{" + + "\"oldNamespace\": \"foo\", " + + "\"newNamespace\": null, " + + "\"removeOldNamespace\": false, " + + "\"mergeIfNewNamespaceAlreadyExists\": true" + + "}"; + + mockMvc.perform(post("/admin/change-namespace") + .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) + .with(csrf().asHeader()) + .content(content) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(content().json(errorJson("New namespace must have a value"))); + } + + @Test + public void testChangeNamespaceNewNamespaceEmpty() throws Exception { + mockAdminUser(); + var content = "{" + + "\"oldNamespace\": \"foo\", " + + "\"newNamespace\": \"\", " + + "\"removeOldNamespace\": false, " + + "\"mergeIfNewNamespaceAlreadyExists\": true" + + "}"; + + mockMvc.perform(post("/admin/change-namespace") + .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) + .with(csrf().asHeader()) + .content(content) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(content().json(errorJson("New namespace must have a value"))); + } + + @Test + public void testChangeNamespaceNewNamespaceAlreadyExists() throws Exception { + mockAdminUser(); + var foo = new Namespace(); + foo.setName("foo"); + Mockito.when(repositories.findNamespace(foo.getName())).thenReturn(foo); + + var bar = new Namespace(); + bar.setName("bar"); + Mockito.when(repositories.findNamespace(bar.getName())).thenReturn(bar); + + var extension = Mockito.mock(Extension.class); + Mockito.when(repositories.findExtensions(foo)).thenReturn(Streamable.of(extension)); + + var membership = Mockito.mock(NamespaceMembership.class); + Mockito.when(repositories.findMemberships(foo)).thenReturn(Streamable.of(membership)); + + var content = "{" + + "\"oldNamespace\": \"foo\", " + + "\"newNamespace\": \"bar\", " + + "\"removeOldNamespace\": false, " + + "\"mergeIfNewNamespaceAlreadyExists\": true" + + "}"; + + mockMvc.perform(post("/admin/change-namespace") + .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) + .with(csrf().asHeader()) + .content(content) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().json(successJson("Changed namespace foo to bar"))) + .andExpect(result -> Mockito.verify(extension).setNamespace(bar)) + .andExpect(result -> Mockito.verify(membership).setNamespace(bar)); + } + + @Test + public void testChangeNamespaceRemoveOldNamespace() throws Exception { + mockAdminUser(); + var foo = new Namespace(); + foo.setName("foo"); + Mockito.when(repositories.findNamespace(foo.getName())).thenReturn(foo); + + var bar = new Namespace(); + bar.setName("bar"); + Mockito.when(repositories.findNamespace(bar.getName())).thenReturn(bar); + + var extension = Mockito.mock(Extension.class); + Mockito.when(repositories.findExtensions(foo)).thenReturn(Streamable.of(extension)); + + var membership = Mockito.mock(NamespaceMembership.class); + Mockito.when(repositories.findMemberships(foo)).thenReturn(Streamable.of(membership)); + + var content = "{" + + "\"oldNamespace\": \"foo\", " + + "\"newNamespace\": \"bar\", " + + "\"removeOldNamespace\": true, " + + "\"mergeIfNewNamespaceAlreadyExists\": true" + + "}"; + + mockMvc.perform(post("/admin/change-namespace") + .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) + .with(csrf().asHeader()) + .content(content) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().json(successJson("Changed namespace foo to bar"))) + .andExpect(result -> Mockito.verify(extension).setNamespace(bar)) + .andExpect(result -> Mockito.verify(membership).setNamespace(bar)) + .andExpect(result -> Mockito.verify(entityManager).remove(foo)); + } + + @Test + public void testChangeNamespaceAbortOnNewNamespaceExists() throws Exception { + mockAdminUser(); + var foo = new Namespace(); + foo.setName("foo"); + Mockito.when(repositories.findNamespace(foo.getName())).thenReturn(foo); + + var bar = new Namespace(); + bar.setName("bar"); + Mockito.when(repositories.findNamespace(bar.getName())).thenReturn(bar); + + var content = "{" + + "\"oldNamespace\": \"foo\", " + + "\"newNamespace\": \"bar\", " + + "\"removeOldNamespace\": false, " + + "\"mergeIfNewNamespaceAlreadyExists\": false" + + "}"; + + mockMvc.perform(post("/admin/change-namespace") + .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) + .with(csrf().asHeader()) + .content(content) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isBadRequest()) + .andExpect(content().json(errorJson("New namespace already exists: bar"))); + } + + @Test + public void testChangeNamespaceAbortNewNamespaceDoesNotExist() throws Exception { + mockAdminUser(); + var foo = new Namespace(); + foo.setName("foo"); + Mockito.when(repositories.findNamespace(foo.getName())).thenReturn(foo); + + var bar = new Namespace(); + bar.setName("bar"); + Mockito.when(repositories.findNamespace(bar.getName())).thenReturn(null); + + var extension = Mockito.mock(Extension.class); + Mockito.when(repositories.findExtensions(foo)).thenReturn(Streamable.of(extension)); + + var membership = Mockito.mock(NamespaceMembership.class); + Mockito.when(repositories.findMemberships(foo)).thenReturn(Streamable.of(membership)); + + var content = "{" + + "\"oldNamespace\": \"foo\", " + + "\"newNamespace\": \"bar\", " + + "\"removeOldNamespace\": false, " + + "\"mergeIfNewNamespaceAlreadyExists\": false" + + "}"; + + mockMvc.perform(post("/admin/change-namespace") + .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) + .with(csrf().asHeader()) + .content(content) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(content().json(successJson("Changed namespace foo to bar"))) + .andExpect(result -> Mockito.verify(extension).setNamespace(bar)) + .andExpect(result -> Mockito.verify(membership).setNamespace(bar)); + } + //---------- UTILITY ----------// private PersonalAccessToken mockAdminToken() { diff --git a/webui/src/extension-registry-service.ts b/webui/src/extension-registry-service.ts index 53980bbc7..e8436ba45 100644 --- a/webui/src/extension-registry-service.ts +++ b/webui/src/extension-registry-service.ts @@ -441,6 +441,24 @@ export class AdminService { }); } + async changeNamespace(abortController: AbortController, req: {oldNamespace: string, newNamespace: string, removeOldNamespace: boolean, mergeIfNewNamespaceAlreadyExists: boolean}): Promise> { + const csrfToken = await this.registry.getCsrfToken(abortController); + const headers: Record = { + 'Content-Type': 'application/json;charset=UTF-8' + }; + if (!isError(csrfToken)) { + headers[csrfToken.header] = csrfToken.value; + } + return sendRequest({ + abortController, + credentials: true, + endpoint: createAbsoluteURL([this.registry.serverUrl, 'admin', 'change-namespace']), + method: 'POST', + payload: req, + headers + }); + } + async getPublisherInfo(abortController: AbortController, provider: string, login: string): Promise> { return sendRequest({ abortController, diff --git a/webui/src/pages/admin-dashboard/namespace-admin.tsx b/webui/src/pages/admin-dashboard/namespace-admin.tsx index af9871c4f..baf1c7ce6 100644 --- a/webui/src/pages/admin-dashboard/namespace-admin.tsx +++ b/webui/src/pages/admin-dashboard/namespace-admin.tsx @@ -89,6 +89,7 @@ export const NamespaceAdmin: FunctionComponent = props => { setLoadingState={setLoading} namespace={currentNamespace} filterUsers={() => true} + onNamespaceChange={fetchNamespace} fixSelf={false} /> diff --git a/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx b/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx new file mode 100644 index 000000000..d3aeb33be --- /dev/null +++ b/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx @@ -0,0 +1,120 @@ +/******************************************************************************** + * Copyright (c) 2020 TypeFox and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ + + import React, { FunctionComponent, useState, useContext, useEffect } from 'react'; + import { + Button, Checkbox, Dialog, DialogActions, DialogContent, DialogContentText, DialogTitle, FormControlLabel, TextField + } from '@material-ui/core'; + import { ButtonWithProgress } from '../../components/button-with-progress'; + import { Namespace, isError } from '../../extension-registry-types'; + import { MainContext } from '../../context'; + + export interface NamespaceChangeDialogProps { + open: boolean; + onClose: (newNamespaceName?: string) => void; + namespace: Namespace; + setLoadingState: (loading: boolean) => void; + } + + export const NamespaceChangeDialog: FunctionComponent = props => { + const { open } = props; + const { service, handleError } = useContext(MainContext); + const [working, setWorking] = useState(false); + const [newNamespace, setNewNamespace] = useState(''); + const [removeOldNamespace, setRemoveOldNamespace] = useState(false); + const [mergeIfNewNamespaceAlreadyExists, setMergeIfNewNamespaceAlreadyExists] = useState(false); + + const abortController = new AbortController(); + useEffect(() => { + return () => { + abortController.abort(); + }; + }, []); + + useEffect(() => { + if (open) { + setNewNamespace(''); + setRemoveOldNamespace(true); + setMergeIfNewNamespaceAlreadyExists(false); + } + }, [open]); + + const onClose = () => { + props.onClose(); + }; + const onRemoveOldNamespaceChange = (event: React.ChangeEvent, checked: boolean) => { + setRemoveOldNamespace(checked); + }; + const onMergeIfNewNamespaceAlreadyExistsChange = (event: React.ChangeEvent, checked: boolean) => { + setMergeIfNewNamespaceAlreadyExists(checked); + }; + const handleChangeNamespace = async () => { + try { + if (!props.namespace) { + return; + } + setWorking(true); + props.setLoadingState(true); + const oldNamespace = props.namespace.name; + const result = await service.admin.changeNamespace(abortController, { oldNamespace, newNamespace, removeOldNamespace, mergeIfNewNamespaceAlreadyExists }); + if (isError(result)) { + throw result; + } + props.setLoadingState(false); + setWorking(false); + props.onClose(newNamespace); + } catch (err) { + props.setLoadingState(false); + setWorking(false); + handleError(err); + } + }; + + return <> + + Change Namespace + + + Enter the new Namespace name. + + { + setNewNamespace(event.target.value); + }} + /> + } + label={`Remove '${props.namespace.name}' namespace after namespace change`} /> + } + label='Merge namespaces if new namespace already exists' /> + + + + + Change Namespace + + + + ; + }; \ No newline at end of file diff --git a/webui/src/pages/user/user-namespace-member-list.tsx b/webui/src/pages/user/user-namespace-member-list.tsx index 1fa51be1c..cf4c226b0 100644 --- a/webui/src/pages/user/user-namespace-member-list.tsx +++ b/webui/src/pages/user/user-namespace-member-list.tsx @@ -88,7 +88,7 @@ export const UserNamespaceMemberList: FunctionComponent - Members in {props.namespace.name} + Members diff --git a/webui/src/pages/user/user-settings-namespace-detail.tsx b/webui/src/pages/user/user-settings-namespace-detail.tsx index 87aea5164..66db65907 100644 --- a/webui/src/pages/user/user-settings-namespace-detail.tsx +++ b/webui/src/pages/user/user-settings-namespace-detail.tsx @@ -8,11 +8,14 @@ * SPDX-License-Identifier: EPL-2.0 ********************************************************************************/ -import React, { FunctionComponent } from 'react'; -import { makeStyles, Grid, Link, Paper, Box } from '@material-ui/core'; +import React, { useState, FunctionComponent } from 'react'; +import { withRouter, RouteComponentProps } from 'react-router-dom'; +import { makeStyles, Box, Button, Link, Paper, Grid, Typography } from '@material-ui/core'; import WarningIcon from '@material-ui/icons/Warning'; import { UserNamespaceExtensionListContainer } from './user-namespace-extension-list'; +import { AdminDashboardRoutes } from '../admin-dashboard/admin-dashboard'; import { Namespace, UserData } from '../../extension-registry-types'; +import { NamespaceChangeDialog } from '../admin-dashboard/namespace-change-dialog'; import { UserNamespaceMemberList } from './user-namespace-member-list'; import { UserNamespaceDetails } from './user-namespace-details'; @@ -62,11 +65,37 @@ const useStyles = makeStyles((theme) => ({ color: '#fff', textDecoration: 'underline' } + }, + changeButton: { + [theme.breakpoints.down('md')]: { + marginLeft: theme.spacing(2) + } + }, + namespaceHeader: { + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + marginBottom: theme.spacing(1), + [theme.breakpoints.down('sm')]: { + flexDirection: 'column', + alignItems: 'center' + } } })); -export const NamespaceDetail: FunctionComponent = props => { +export const NamespaceDetailComponent: FunctionComponent = props => { const classes = useStyles(); + const [changeDialogIsOpen, setChangeDialogIsOpen] = useState(false); + const handleCloseChangeDialog = async (newNamespaceName?: string) => { + setChangeDialogIsOpen(false); + if (newNamespaceName && props.onNamespaceChange) { + props.onNamespaceChange(newNamespaceName); + } + }; + const handleOpenChangeDialog = () => { + setChangeDialogIsOpen(true); + }; + return <> { @@ -85,6 +114,17 @@ export const NamespaceDetail: FunctionComponent = props = : null } + + + {props.namespace.name} + { props.location.pathname.startsWith(AdminDashboardRoutes.NAMESPACE_ADMIN) + ? + : null + } + + { props.namespace.membersUrl ? @@ -105,16 +145,24 @@ export const NamespaceDetail: FunctionComponent = props = /> + ; }; -export namespace NamespaceDetail { - export interface Props { +export namespace NamespaceDetailComponent { + export interface Props extends RouteComponentProps { namespace: Namespace; filterUsers: (user: UserData) => boolean; + onNamespaceChange?: (newNamespaceName: string) => void; fixSelf: boolean; setLoadingState: (loading: boolean) => void; namespaceAccessUrl?: string; theme?: string; } } + +export const NamespaceDetail = withRouter(NamespaceDetailComponent); From 4544beb3609d04922b9b8dcaf979bf315bfc5af7 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Mon, 20 Feb 2023 09:16:49 +0200 Subject: [PATCH 11/45] Add a way to proxy upstream requests instead of external links Fixes #452 Added UpstreamProxyService Add url rewriting to UpstreamVSCodeService.java # Conflicts: # server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java # server/src/main/java/org/eclipse/openvsx/adapter/UpstreamVSCodeService.java --- .../eclipse/openvsx/UpstreamProxyService.java | 149 ++++++++++++++++++ .../openvsx/UpstreamRegistryService.java | 18 ++- .../adapter/UpstreamVSCodeService.java | 55 ++++++- 3 files changed, 211 insertions(+), 11 deletions(-) create mode 100644 server/src/main/java/org/eclipse/openvsx/UpstreamProxyService.java diff --git a/server/src/main/java/org/eclipse/openvsx/UpstreamProxyService.java b/server/src/main/java/org/eclipse/openvsx/UpstreamProxyService.java new file mode 100644 index 000000000..b12bd3dd2 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/UpstreamProxyService.java @@ -0,0 +1,149 @@ +/******************************************************************************** + * Copyright (c) 2022 Precies. Software and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ +package org.eclipse.openvsx; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import org.eclipse.openvsx.adapter.ExtensionQueryResult; +import org.eclipse.openvsx.json.*; +import org.eclipse.openvsx.util.UrlUtil; +import org.elasticsearch.common.Strings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Component +@ConditionalOnProperty(value="ovsx.upstream.proxy.enabled", havingValue = "true") +public class UpstreamProxyService { + + protected final Logger logger = LoggerFactory.getLogger(UpstreamProxyService.class); + + public NamespaceJson rewriteUrls(NamespaceJson json) { + rewriteUrlMap(json.extensions); + if(!Strings.isNullOrEmpty(json.membersUrl)) { + json.membersUrl = rewriteUrl(json.membersUrl); + } + if(!Strings.isNullOrEmpty(json.roleUrl)) { + json.roleUrl = rewriteUrl(json.roleUrl); + } + + return json; + } + + public ExtensionJson rewriteUrls(ExtensionJson json) { + json.namespaceUrl = rewriteUrl(json.namespaceUrl); + json.reviewsUrl = rewriteUrl(json.reviewsUrl); + rewriteUrlMap(json.files); + rewriteUrlMap(json.allVersions); + json.dependencies = rewriteUrlList(json.dependencies, this::rewriteUrls); + json.bundledExtensions = rewriteUrlList(json.bundledExtensions, this::rewriteUrls); + rewriteUrlMap(json.downloads); + return json; + } + + public SearchResultJson rewriteUrls(SearchResultJson json) { + json.extensions = rewriteUrlList(json.extensions, this::rewriteUrls); + return json; + } + + public QueryResultJson rewriteUrls(QueryResultJson json) { + json.extensions = rewriteUrlList(json.extensions, this::rewriteUrls); + return json; + } + + public ExtensionQueryResult rewriteUrls(ExtensionQueryResult json) { + for(var result : json.results) { + for(var extension : result.extensions) { + for(var version : extension.versions) { + version.assetUri = rewriteUrl(version.assetUri); + version.fallbackAssetUri = rewriteUrl(version.fallbackAssetUri); + for (var file : version.files) { + file.source = rewriteUrl(file.source); + } + } + } + } + return json; + } + + public JsonNode rewriteUrls(JsonNode json) { + if(json.isArray()) { + var list = new ObjectMapper().createArrayNode(); + var array = (ArrayNode) json; + array.forEach(url -> list.add(rewriteUrl(url.asText()))); + json = list; + } + + return json; + } + + public URI rewriteUrl(URI location) { + return URI.create(rewriteUrl(location.toString())); + } + + private SearchEntryJson rewriteUrls(SearchEntryJson json) { + json.url = rewriteUrl(json.url); + rewriteUrlMap(json.files); + json.allVersions = rewriteUrlList(json.allVersions, this::rewriteUrls); + + return json; + } + + private SearchEntryJson.VersionReference rewriteUrls(SearchEntryJson.VersionReference json) { + json.url = rewriteUrl(json.url); + rewriteUrlMap(json.files); + + return json; + } + + private ExtensionReferenceJson rewriteUrls(ExtensionReferenceJson json) { + json.url = rewriteUrl(json.url); + return json; + } + + private List rewriteUrlList(List jsonList, Function mapper) { + return jsonList != null ? jsonList.stream().map(mapper).collect(Collectors.toList()) : jsonList; + } + + private void rewriteUrlMap(Map map) { + if(map != null) { + map.replaceAll((k, v) -> rewriteUrl(v)); + } + } + + private String rewriteUrl(String url) { + var baseUri = URI.create(UrlUtil.getBaseUrl()); + var uri = URI.create(url); + + var scheme = baseUri.getScheme(); + var userInfo = baseUri.getUserInfo(); + var host = baseUri.getHost(); + var port = baseUri.getPort(); + var path = uri.getPath(); + var query = uri.getQuery(); + var fragment = uri.getFragment(); + + try { + return new URI(scheme, userInfo, host, port, path, query, fragment).toString(); + } catch (URISyntaxException e) { + logger.error("failed to rewrite URI: {}", uri); + return null; + } + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java b/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java index 0333760b2..c3842dd97 100644 --- a/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java @@ -41,6 +41,9 @@ public class UpstreamRegistryService implements IExtensionRegistry { @Autowired RestTemplate restTemplate; + @Autowired(required = false) + UpstreamProxyService proxy; + @Autowired UrlConfigService urlConfigService; @@ -53,7 +56,8 @@ public NamespaceJson getNamespace(String namespace) { var urlTemplate = urlConfigService.getUpstreamUrl() + "/api/{namespace}"; var uriVariables = Map.of("namespace", namespace); try { - return restTemplate.getForObject(urlTemplate, NamespaceJson.class, uriVariables); + var json = restTemplate.getForObject(urlTemplate, NamespaceJson.class, uriVariables); + return proxy != null ? proxy.rewriteUrls(json) : json; } catch (RestClientException exc) { if(!isNotFound(exc)) { var url = UriComponentsBuilder.fromUriString(urlTemplate).build(uriVariables); @@ -96,7 +100,7 @@ public ExtensionJson getExtension(String namespace, String extension, String tar try { var json = restTemplate.getForObject(urlTemplate, ExtensionJson.class, uriVariables); makeDownloadsCompatible(json); - return json; + return proxy != null ? proxy.rewriteUrls(json) : json; } catch (RestClientException exc) { if(!isNotFound(exc)) { var url = UriComponentsBuilder.fromUriString(urlTemplate).build(uriVariables); @@ -124,7 +128,7 @@ public ExtensionJson getExtension(String namespace, String extension, String tar try { var json = restTemplate.getForObject(urlTemplate, ExtensionJson.class, uriVariables); makeDownloadsCompatible(json); - return json; + return proxy != null ? proxy.rewriteUrls(json) : json; } catch (RestClientException exc) { if(!isNotFound(exc)) { var url = UriComponentsBuilder.fromUriString(urlTemplate).build(uriVariables); @@ -224,7 +228,8 @@ public SearchResultJson search(ISearchService.Options options) { } try { - return restTemplate.getForObject(urlTemplate, SearchResultJson.class, uriVariables); + var json = restTemplate.getForObject(urlTemplate, SearchResultJson.class, uriVariables); + return proxy != null ? proxy.rewriteUrls(json) : json; } catch (RestClientException exc) { if(!isNotFound(exc)) { var url = UriComponentsBuilder.fromUriString(urlTemplate).build(uriVariables); @@ -258,7 +263,8 @@ public QueryResultJson query(QueryParamJson param) { } try { - return restTemplate.getForObject(urlTemplate, QueryResultJson.class, queryParams); + var json = restTemplate.getForObject(urlTemplate, QueryResultJson.class, queryParams); + return proxy != null ? proxy.rewriteUrls(json) : json; } catch (RestClientException exc) { if(!isNotFound(exc)) { var url = UriComponentsBuilder.fromUriString(urlTemplate).build(queryParams); @@ -315,7 +321,7 @@ private void handleError(Throwable exc) throws RuntimeException { } private void makeDownloadsCompatible(ExtensionJson json) { - if(json.downloads == null && json.files.containsKey("download")) { + if (json.downloads == null && json.files.containsKey("download")) { json.downloads = new HashMap<>(); json.downloads.put(TargetPlatform.NAME_UNIVERSAL, json.files.get("download")); } diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/UpstreamVSCodeService.java b/server/src/main/java/org/eclipse/openvsx/adapter/UpstreamVSCodeService.java index ad6846910..08e7a8328 100644 --- a/server/src/main/java/org/eclipse/openvsx/adapter/UpstreamVSCodeService.java +++ b/server/src/main/java/org/eclipse/openvsx/adapter/UpstreamVSCodeService.java @@ -9,7 +9,9 @@ * ****************************************************************************** */ package org.eclipse.openvsx.adapter; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; +import org.eclipse.openvsx.UpstreamProxyService; import org.eclipse.openvsx.UrlConfigService; import org.eclipse.openvsx.util.HttpHeadersUtil; import org.eclipse.openvsx.util.NotFoundException; @@ -23,7 +25,9 @@ import org.springframework.web.client.RestTemplate; import org.springframework.web.util.UriComponentsBuilder; +import java.io.IOException; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; @@ -35,6 +39,9 @@ public class UpstreamVSCodeService implements IVSCodeService { @Autowired RestTemplate restTemplate; + @Autowired(required = false) + UpstreamProxyService proxy; + @Autowired RestTemplate nonRedirectingRestTemplate; @@ -58,7 +65,8 @@ public ExtensionQueryResult extensionQuery(ExtensionQueryParam param, int defaul var statusCode = response.getStatusCode(); if(statusCode.is2xxSuccessful()) { - return response.getBody(); + var json = response.getBody(); + return proxy != null ? proxy.rewriteUrls(json) : json; } if(statusCode.isError() && statusCode != HttpStatus.NOT_FOUND) { logger.error("POST {}: {}", urlTemplate, response); @@ -99,7 +107,23 @@ public ResponseEntity browse(String namespaceName, String extensionName, headers.addAll(response.getHeaders()); headers.remove(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN); headers.remove(HttpHeaders.VARY); - return new ResponseEntity<>(response.getBody(), headers, response.getStatusCode()); + + if(proxy != null && MediaType.APPLICATION_JSON.equals(headers.getContentType())) { + try { + var mapper = new ObjectMapper(); + var json = mapper.readTree(response.getBody()); + json = proxy.rewriteUrls(json); + response = ResponseEntity.status(statusCode) + .headers(headers) + .body(mapper.writeValueAsString(json).getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + logger.error("Failed to read/write JSON", e); + } + } else { + response = new ResponseEntity<>(response.getBody(), headers, response.getStatusCode()); + } + + return response; } if(statusCode.isError() && statusCode != HttpStatus.NOT_FOUND) { var url = UriComponentsBuilder.fromUriString(urlTemplate).build(uriVariables); @@ -129,7 +153,12 @@ public String download(String namespace, String extension, String version, Strin var statusCode = response.getStatusCode(); if(statusCode.is3xxRedirection()) { - return response.getHeaders().getLocation().toString(); + var location = response.getHeaders().getLocation(); + if(proxy != null) { + location = proxy.rewriteUrl(location); + } + + return location.toString(); } if(statusCode.isError() && statusCode != HttpStatus.NOT_FOUND) { var url = UriComponentsBuilder.fromUriString(urlTemplate).build(uriVariables); @@ -154,7 +183,12 @@ public String getItemUrl(String namespace, String extension) { var statusCode = response.getStatusCode(); if(statusCode.is3xxRedirection()) { - return response.getHeaders().getLocation().toString(); + var location = response.getHeaders().getLocation(); + if(proxy != null) { + location = proxy.rewriteUrl(location); + } + + return location.toString(); } if(statusCode.isError() && statusCode != HttpStatus.NOT_FOUND) { var url = UriComponentsBuilder.fromUriString(urlTemplate).build(uriVariables); @@ -194,13 +228,24 @@ public ResponseEntity getAsset(String namespace, String extensionName, S } var statusCode = response.getStatusCode(); - if(statusCode.is2xxSuccessful() || statusCode.is3xxRedirection()) { + if(statusCode.is2xxSuccessful()) { var headers = new HttpHeaders(); headers.addAll(response.getHeaders()); headers.remove(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN); headers.remove(HttpHeaders.VARY); return new ResponseEntity<>(response.getBody(), headers, response.getStatusCode()); } + if(statusCode.is3xxRedirection()) { + var location = response.getHeaders().getLocation(); + if(proxy != null) { + location = proxy.rewriteUrl(location); + } + + return ResponseEntity.status(HttpStatus.FOUND) + .headers(response.getHeaders()) + .location(location) + .build(); + } if(statusCode.isError() && statusCode != HttpStatus.NOT_FOUND) { var url = UriComponentsBuilder.fromUriString(urlTemplate).build(uriVariables); logger.error("GET {}: {}", url, response); From 0862c24f9dca1b07151349a9e6e6618a61a63e27 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Wed, 22 Feb 2023 14:07:07 +0200 Subject: [PATCH 12/45] Insert unique extension public id Update extension publicId of already published extensions when a new extension has the same publicId --- .../openvsx/adapter/VSCodeIdService.java | 21 ++------------- .../PublishExtensionVersionHandler.java | 26 ++++++++++++++++++- .../PublishExtensionVersionService.java | 6 +++++ 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java index f4fb99efc..4e491389d 100644 --- a/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java +++ b/server/src/main/java/org/eclipse/openvsx/adapter/VSCodeIdService.java @@ -9,13 +9,8 @@ ********************************************************************************/ package org.eclipse.openvsx.adapter; -import java.util.UUID; - -import javax.transaction.Transactional; - import com.google.common.base.Strings; import com.google.common.collect.Lists; - import org.eclipse.openvsx.UrlConfigService; import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.repositories.RepositoryService; @@ -29,6 +24,8 @@ import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; +import java.util.UUID; + @Component public class VSCodeIdService { @@ -55,7 +52,6 @@ public boolean setPublicIds(Extension extension) { } if (upstream.publisher != null && upstream.publisher.publisherId != null) { extension.getNamespace().setPublicId(upstream.publisher.publisherId); - updateExistingPublicIds = true; } } if (extension.getPublicId() == null) { @@ -68,19 +64,6 @@ public boolean setPublicIds(Extension extension) { return updateExistingPublicIds; } - @Transactional - public void updateExistingPublicIds(Extension extension) { - var existingExtension = repositories.findExtensionByPublicId(extension.getPublicId()); - if(existingExtension != null && !existingExtension.equals(extension)) { - existingExtension.setPublicId(createRandomId()); - } - - var existingNamespace = repositories.findNamespaceByPublicId(extension.getNamespace().getPublicId()); - if(existingNamespace != null && !existingNamespace.equals(extension.getNamespace())) { - existingNamespace.setPublicId(createRandomId()); - } - } - private String createRandomId() { return UUID.randomUUID().toString(); } diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java index 02c8dbdd8..eab2479b3 100644 --- a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionHandler.java @@ -32,6 +32,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -112,7 +115,7 @@ private ExtensionVersion createExtensionVersion(ExtensionProcessor processor, Us var updateExistingPublicIds = vsCodeIdService.setPublicIds(extension); if(updateExistingPublicIds) { - vsCodeIdService.updateExistingPublicIds(extension); + updateExistingPublicIds(extension).forEach(service::updateExtensionPublicId); } entityManager.persist(extension); @@ -167,6 +170,27 @@ private String checkBundledExtension(String bundledExtension) { return bundledExtension; } + private List updateExistingPublicIds(Extension extension) { + var updated = true; + var updatedExtensions = new ArrayList(); + var newExtension = extension; + while(updated) { + updated = false; + var oldExtension = repositories.findExtensionByPublicId(newExtension.getPublicId()); + if (oldExtension != null && !oldExtension.equals(newExtension)) { + entityManager.detach(oldExtension); + updated = vsCodeIdService.setPublicIds(oldExtension); + } + if(updated) { + updatedExtensions.add(oldExtension); + newExtension = oldExtension; + } + } + + Collections.reverse(updatedExtensions); + return updatedExtensions; + } + @Async @Retryable public void publishAsync(FileResource download, Path extensionFile, ExtensionService extensionService) { diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java index 4009eab9e..d44ad3a60 100644 --- a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java @@ -17,6 +17,7 @@ import javax.transaction.Transactional; import org.eclipse.openvsx.ExtensionService; +import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.FileResource; import org.eclipse.openvsx.repositories.RepositoryService; @@ -88,4 +89,9 @@ public void activateExtension(ExtensionVersion extVersion, ExtensionService exte extVersion = entityManager.merge(extVersion); extensions.updateExtension(extVersion.getExtension()); } + + @Transactional(Transactional.TxType.REQUIRES_NEW) + public void updateExtensionPublicId(Extension extension) { + entityManager.merge(extension); + } } From a9d453d12e7e519ef18bb16b9d49c08020c82c12 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Tue, 21 Feb 2023 22:24:11 +0200 Subject: [PATCH 13/45] website: improve information from search result Use react-helmet to make tags configurable through pageSettings for homepage, extension and namespace pages Simplify header tags Pass params instead of only name --- webui/package.json | 6 +- webui/src/default/default-app.tsx | 7 +- webui/src/default/page-settings.tsx | 35 +- webui/src/page-settings.ts | 11 +- .../extension-detail/extension-detail.tsx | 56 +- .../extension-list-container.tsx | 1 - .../namespace-detail/namespace-detail.tsx | 54 +- .../src/pages/user/generate-token-dialog.tsx | 4 +- webui/src/pages/user/user-settings.tsx | 66 +- webui/yarn.lock | 570 +++++++++--------- 10 files changed, 452 insertions(+), 358 deletions(-) diff --git a/webui/package.json b/webui/package.json index 649a82150..40dbe7d90 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "openvsx-webui", - "version": "0.8.0", + "version": "0.9.0", "description": "User interface for Eclipse Open VSX", "keywords": [ "react", @@ -49,6 +49,7 @@ "react-avatar-editor": "^13.0.0", "react-dom": "^16.14.0", "react-dropzone": "^14.2.1", + "react-helmet": "^6.1.0", "react-infinite-scroller": "^1.2.4", "react-router-dom": "^5.2.0", "react-truncate": "^2.4.0" @@ -69,6 +70,7 @@ "@types/react": "^16.9.34", "@types/react-avatar-editor": "^13.0.0", "@types/react-dom": "^16.9.6", + "@types/react-helmet": "^6.1.6", "@types/react-infinite-scroller": "^1.2.1", "@types/react-router-dom": "*", "@types/react-truncate": "^2.3.4", @@ -84,7 +86,7 @@ "style-loader": "^2.0.0", "ts-mocha": "^10.0.0", "typescript": "~4.1.2", - "webpack": "^5.9.0", + "webpack": "^5.75.0", "webpack-cli": "^4.2.0" }, "scripts": { diff --git a/webui/src/default/default-app.tsx b/webui/src/default/default-app.tsx index 5a6326b32..bb9cb30ab 100644 --- a/webui/src/default/default-app.tsx +++ b/webui/src/default/default-app.tsx @@ -40,15 +40,16 @@ const App = () => { ); const pageSettings = createPageSettings(theme, prefersDarkMode, service.serverUrl); - - return ( + const { mainHeadTags: MainHeadTagsComponent } = pageSettings.elements; + return (<> + { MainHeadTagsComponent ? : null}
- ); + ); }; const node = document.getElementById('main'); diff --git a/webui/src/default/page-settings.tsx b/webui/src/default/page-settings.tsx index ba1961494..c00d75e07 100644 --- a/webui/src/default/page-settings.tsx +++ b/webui/src/default/page-settings.tsx @@ -9,12 +9,16 @@ ********************************************************************************/ import * as React from 'react'; +import { Helmet } from 'react-helmet'; import { makeStyles } from '@material-ui/styles'; import { Link, Typography, Theme, Box } from '@material-ui/core'; import { Link as RouteLink, Route } from 'react-router-dom'; import GitHubIcon from '@material-ui/icons/GitHub'; +import { Extension, NamespaceDetails } from '../extension-registry-types'; import { PageSettings } from '../page-settings'; import { ExtensionListRoutes } from '../pages/extension-list/extension-list-container'; +import { ExtensionDetailComponent } from '../pages/extension-detail/extension-detail'; +import { NamespaceDetailComponent } from '../pages/namespace-detail/namespace-detail'; import { DefaultMenuContent, MobileMenuContent } from './menu-content'; import OpenVSXLogo from './openvsx-registry-logo'; import About from './about'; @@ -89,6 +93,32 @@ export default function createPageSettings(theme: Theme, prefersDarkMode: boolea const additionalRoutes: React.FunctionComponent = () => } />; + const headTags: React.FunctionComponent<{title: string}> = (props) => { + return + {props.title} + ; + }; + + const mainHeadTags: React.FunctionComponent<{pageSettings: PageSettings}> = (props) => { + return headTags({ title: props.pageSettings.pageTitle }); + }; + + const extensionHeadTags: React.FunctionComponent<{extension?: Extension, params: ExtensionDetailComponent.Params, pageSettings: PageSettings}> = (props) => { + const name = props.extension + ? props.extension.displayName || props.extension.name + : props.params.name; + + return headTags({ title: `${name} – ${props.pageSettings.pageTitle}` }); + }; + + const namespaceHeadTags: React.FunctionComponent<{namespaceDetails?: NamespaceDetails, params: NamespaceDetailComponent.Params, pageSettings: PageSettings}> = (props) => { + const name = props.namespaceDetails + ? props.namespaceDetails.displayName || props.namespaceDetails.name + : props.params.name; + + return headTags({ title: `${name} – ${props.pageSettings.pageTitle}` }); + }; + return { pageTitle: 'Open VSX Registry', themeType: prefersDarkMode ? 'dark' : 'light', @@ -103,7 +133,10 @@ export default function createPageSettings(theme: Theme, prefersDarkMode: boolea } }, searchHeader, - additionalRoutes + additionalRoutes, + mainHeadTags, + extensionHeadTags, + namespaceHeadTags }, urls: { extensionDefaultIcon: '/default-icon.png', diff --git a/webui/src/page-settings.ts b/webui/src/page-settings.ts index c197664a1..90b6b5fe4 100644 --- a/webui/src/page-settings.ts +++ b/webui/src/page-settings.ts @@ -9,7 +9,9 @@ ********************************************************************************/ import * as React from 'react'; -import { Extension } from './extension-registry-types'; +import { Extension, NamespaceDetails } from './extension-registry-types'; +import { ExtensionDetailComponent } from './pages/extension-detail/extension-detail'; +import { NamespaceDetailComponent } from './pages/namespace-detail/namespace-detail'; import { Cookie } from './utils'; export interface PageSettings { @@ -41,13 +43,16 @@ export interface PageSettings { color?: 'info' | 'warning' }, cookie?: Cookie - } + }; + mainHeadTags?: React.ComponentType<{ pageSettings: PageSettings }>; + extensionHeadTags?: React.ComponentType<{ extension?: Extension, params: ExtensionDetailComponent.Params, pageSettings: PageSettings }>; + namespaceHeadTags?: React.ComponentType<{ namespaceDetails?: NamespaceDetails, params: NamespaceDetailComponent.Params, pageSettings: PageSettings }>; }; urls: { extensionDefaultIcon: string; namespaceAccessInfo: string; publisherAgreement?: string; - } + }; } export interface Styleable { diff --git a/webui/src/pages/extension-detail/extension-detail.tsx b/webui/src/pages/extension-detail/extension-detail.tsx index d3a07a925..f0cfb0a93 100644 --- a/webui/src/pages/extension-detail/extension-detail.tsx +++ b/webui/src/pages/extension-detail/extension-detail.tsx @@ -170,7 +170,6 @@ export class ExtensionDetailComponent extends React.Component - - { - this.state.notFoundError ? - - - {this.state.notFoundError} - - - : null - } - ; - } + const params = this.props.match.params as ExtensionDetailComponent.Params; + return <> + { this.renderHeaderTags(params, extension) } + + { + extension + ? this.renderExtension(extension, icon) + : this.renderNotFound() + } + ; + } + + protected renderHeaderTags(params: ExtensionDetailComponent.Params, extension?: Extension): React.ReactNode { + const pageSettings = this.context.pageSettings; + const { extensionHeadTags: ExtensionHeadTagsComponent } = pageSettings.elements; + return + { ExtensionHeadTagsComponent + ? + : null + } + ; + } + + protected renderNotFound(): React.ReactNode { + return + { + this.state.notFoundError ? + + + {this.state.notFoundError} + + + : null + } + ; + } + + protected renderExtension(extension: Extension, icon?: string): React.ReactNode { const classes = this.props.classes; const headerTheme = extension.galleryTheme || this.context.pageSettings.themeType || 'light'; - return <> - - - { - this.state.notFoundError ? - - - {this.state.notFoundError} - - - : null - } - ; - } + const params = this.props.match.params as NamespaceDetailComponent.Params; + return <> + { this.renderHeaderTags(params, namespaceDetails) } + + { + namespaceDetails + ? this.renderNamespaceDetails(namespaceDetails, truncateReadMore) + : this.renderNotFound() + } + ; + } + protected renderHeaderTags(params: NamespaceDetailComponent.Params, namespaceDetails?: NamespaceDetails): React.ReactNode { + const pageSettings = this.context.pageSettings; + const { namespaceHeadTags: NamespaceHeadTagsComponent } = pageSettings.elements; + return + { NamespaceHeadTagsComponent + ? + : null + } + ; + } + + protected renderNotFound(): React.ReactNode { + return <> + { + this.state.notFoundError ? + + + {this.state.notFoundError} + + + : null + } + ; + } + + protected renderNamespaceDetails(namespaceDetails: NamespaceDetails, truncateReadMore: boolean): React.ReactNode { const classes = this.props.classes; return <> - diff --git a/webui/src/pages/user/generate-token-dialog.tsx b/webui/src/pages/user/generate-token-dialog.tsx index 14f66c379..cf65b5a8f 100644 --- a/webui/src/pages/user/generate-token-dialog.tsx +++ b/webui/src/pages/user/generate-token-dialog.tsx @@ -45,8 +45,8 @@ class GenerateTokenDialogComponent extends React.Component { - this.setState({ - open: true, + this.setState({ + open: true, posted: false, description: '', token: undefined diff --git a/webui/src/pages/user/user-settings.tsx b/webui/src/pages/user/user-settings.tsx index 47cfcaefe..ffdf5ab43 100644 --- a/webui/src/pages/user/user-settings.tsx +++ b/webui/src/pages/user/user-settings.tsx @@ -9,6 +9,7 @@ ********************************************************************************/ import * as React from 'react'; +import { Helmet } from 'react-helmet'; import { createStyles, Theme, WithStyles, withStyles, Grid, Container, Box, Typography, Link } from '@material-ui/core'; import { RouteComponentProps, Route } from 'react-router-dom'; import { createRoute } from '../../utils'; @@ -57,14 +58,11 @@ class UserSettingsComponent extends React.Component static contextType = MainContext; declare context: MainContext; - componentDidMount() { - document.title = `Settings – ${this.context.pageSettings.pageTitle}`; - } - - render() { + protected renderContent(): React.ReactNode { if (this.props.userLoading) { return ; } + const user = this.context.user; if (!user) { return @@ -79,32 +77,40 @@ class UserSettingsComponent extends React.Component ; } - return - - - - - - - - - - - - - - - - - - - - - - + + return + + + + - - + + + + + + + + + + + + + + + + + + + ; + } + + render() { + return + + Settings – {this.context.pageSettings.pageTitle} + + { this.renderContent() } ; } } diff --git a/webui/yarn.lock b/webui/yarn.lock index 5a2c73489..20e295bba 100644 --- a/webui/yarn.lock +++ b/webui/yarn.lock @@ -162,7 +162,7 @@ resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== -"@jridgewell/trace-mapping@^0.3.9": +"@jridgewell/trace-mapping@^0.3.14", "@jridgewell/trace-mapping@^0.3.9": version "0.3.17" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz#793041277af9073b0951a7fe0f0d8c4c98c36985" integrity sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g== @@ -324,26 +324,31 @@ dependencies: "@types/trusted-types" "*" -"@types/eslint-scope@^3.7.0": - version "3.7.0" - resolved "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.0.tgz" - integrity sha512-O/ql2+rrCUe2W2rs7wMR+GqPRcgB6UiqN5RhrR5xruFlY7l9YLMn0ZkDzjoHLeiFkR8MCQZVudUuuvQ2BLC9Qw== +"@types/eslint-scope@^3.7.3": + version "3.7.4" + resolved "https://registry.yarnpkg.com/@types/eslint-scope/-/eslint-scope-3.7.4.tgz#37fc1223f0786c39627068a12e94d6e6fc61de16" + integrity sha512-9K4zoImiZc3HlIp6AVUDE4CWYx22a+lhSZMYNpbjW04+YF0KWj4pJXnEMjdnFTiQibFFmElcsasJXDbdI/EPhA== dependencies: "@types/eslint" "*" "@types/estree" "*" "@types/eslint@*": - version "7.2.5" - resolved "https://registry.npmjs.org/@types/eslint/-/eslint-7.2.5.tgz" - integrity sha512-Dc6ar9x16BdaR3NSxSF7T4IjL9gxxViJq8RmFd+2UAyA+K6ck2W+gUwfgpG/y9TPyUuBL35109bbULpEynvltA== + version "8.21.1" + resolved "https://registry.yarnpkg.com/@types/eslint/-/eslint-8.21.1.tgz#110b441a210d53ab47795124dbc3e9bb993d1e7c" + integrity sha512-rc9K8ZpVjNcLs8Fp0dkozd5Pt2Apk1glO4Vgz8ix1u6yFByxfqo5Yavpy65o+93TAe24jr7v+eSBtFLvOQtCRQ== dependencies: "@types/estree" "*" "@types/json-schema" "*" -"@types/estree@*", "@types/estree@^0.0.45": - version "0.0.45" - resolved "https://registry.npmjs.org/@types/estree/-/estree-0.0.45.tgz" - integrity sha512-jnqIUKDUqJbDIUxm0Uj7bnlMnRm1T/eZ9N+AVMqhPgzrba2GhGG5o/jCTwmdPK709nEZsGoMzXEDUjcXHa3W0g== +"@types/estree@*": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.0.tgz#5fb2e536c1ae9bf35366eed879e827fa59ca41c2" + integrity sha512-WulqXMDUTYAXCjZnk6JtIHPigp55cVtDgDrO2gHRwhyJto21+1zbVCtOYB2L1F9w4qCQ0rOGWBnBe0FNTiEJIQ== + +"@types/estree@^0.0.51": + version "0.0.51" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-0.0.51.tgz#cfd70924a25a3fd32b218e5e420e6897e1ac4f40" + integrity sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ== "@types/express-serve-static-core@*": version "4.17.9" @@ -369,16 +374,16 @@ resolved "https://registry.npmjs.org/@types/history/-/history-4.7.5.tgz" integrity sha512-wLD/Aq2VggCJXSjxEwrMafIP51Z+13H78nXIX0ABEuIGhmB5sNGbR113MOKo+yfw+RDo1ZU3DM6yfnnRF/+ouw== -"@types/json-schema@*", "@types/json-schema@^7.0.6": - version "7.0.6" - resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.6.tgz" - integrity sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw== - -"@types/json-schema@^7.0.9": +"@types/json-schema@*", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9": version "7.0.11" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== +"@types/json-schema@^7.0.6": + version "7.0.6" + resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.6.tgz" + integrity sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw== + "@types/json5@^0.0.29": version "0.0.29" resolved "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz" @@ -476,6 +481,13 @@ dependencies: "@types/react" "*" +"@types/react-helmet@^6.1.6": + version "6.1.6" + resolved "https://registry.yarnpkg.com/@types/react-helmet/-/react-helmet-6.1.6.tgz#7d1afd8cbf099616894e8240e9ef70e3c6d7506d" + integrity sha512-ZKcoOdW/Tg+kiUbkFCBtvDw0k3nD4HJ/h/B9yWxN4uDO8OkRksWTO+EL+z/Qu3aHTeTll3Ro0Cc/8UhwBCMG5A== + dependencies: + "@types/react" "*" + "@types/react-infinite-scroller@^1.2.1": version "1.2.1" resolved "https://registry.npmjs.org/@types/react-infinite-scroller/-/react-infinite-scroller-1.2.1.tgz" @@ -623,149 +635,125 @@ "@typescript-eslint/types" "5.44.0" eslint-visitor-keys "^3.3.0" -"@webassemblyjs/ast@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz" - integrity sha512-C6wW5L+b7ogSDVqymbkkvuW9kruN//YisMED04xzeBBqjHa2FYnmvOlS6Xj68xWQRgWvI9cIglsjFowH/RJyEA== +"@webassemblyjs/ast@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.1.tgz#2bfd767eae1a6996f432ff7e8d7fc75679c0b6a7" + integrity sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw== dependencies: - "@webassemblyjs/helper-module-context" "1.9.0" - "@webassemblyjs/helper-wasm-bytecode" "1.9.0" - "@webassemblyjs/wast-parser" "1.9.0" - -"@webassemblyjs/floating-point-hex-parser@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.9.0.tgz" - integrity sha512-TG5qcFsS8QB4g4MhrxK5TqfdNe7Ey/7YL/xN+36rRjl/BlGE/NcBvJcqsRgCP6Z92mRE+7N50pRIi8SmKUbcQA== - -"@webassemblyjs/helper-api-error@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.9.0.tgz" - integrity sha512-NcMLjoFMXpsASZFxJ5h2HZRcEhDkvnNFOAKneP5RbKRzaWJN36NC4jqQHKwStIhGXu5mUWlUUk7ygdtrO8lbmw== + "@webassemblyjs/helper-numbers" "1.11.1" + "@webassemblyjs/helper-wasm-bytecode" "1.11.1" -"@webassemblyjs/helper-buffer@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.9.0.tgz" - integrity sha512-qZol43oqhq6yBPx7YM3m9Bv7WMV9Eevj6kMi6InKOuZxhw+q9hOkvq5e/PpKSiLfyetpaBnogSbNCfBwyB00CA== +"@webassemblyjs/floating-point-hex-parser@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz#f6c61a705f0fd7a6aecaa4e8198f23d9dc179e4f" + integrity sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ== -"@webassemblyjs/helper-code-frame@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.9.0.tgz" - integrity sha512-ERCYdJBkD9Vu4vtjUYe8LZruWuNIToYq/ME22igL+2vj2dQ2OOujIZr3MEFvfEaqKoVqpsFKAGsRdBSBjrIvZA== - dependencies: - "@webassemblyjs/wast-printer" "1.9.0" +"@webassemblyjs/helper-api-error@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz#1a63192d8788e5c012800ba6a7a46c705288fd16" + integrity sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg== -"@webassemblyjs/helper-fsm@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/helper-fsm/-/helper-fsm-1.9.0.tgz" - integrity sha512-OPRowhGbshCb5PxJ8LocpdX9Kl0uB4XsAjl6jH/dWKlk/mzsANvhwbiULsaiqT5GZGT9qinTICdj6PLuM5gslw== +"@webassemblyjs/helper-buffer@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz#832a900eb444884cde9a7cad467f81500f5e5ab5" + integrity sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA== -"@webassemblyjs/helper-module-context@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/helper-module-context/-/helper-module-context-1.9.0.tgz" - integrity sha512-MJCW8iGC08tMk2enck1aPW+BE5Cw8/7ph/VGZxwyvGbJwjktKkDK7vy7gAmMDx88D7mhDTCNKAW5tED+gZ0W8g== +"@webassemblyjs/helper-numbers@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz#64d81da219fbbba1e3bd1bfc74f6e8c4e10a62ae" + integrity sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ== dependencies: - "@webassemblyjs/ast" "1.9.0" + "@webassemblyjs/floating-point-hex-parser" "1.11.1" + "@webassemblyjs/helper-api-error" "1.11.1" + "@xtuc/long" "4.2.2" -"@webassemblyjs/helper-wasm-bytecode@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.9.0.tgz" - integrity sha512-R7FStIzyNcd7xKxCZH5lE0Bqy+hGTwS3LJjuv1ZVxd9O7eHCedSdrId/hMOd20I+v8wDXEn+bjfKDLzTepoaUw== +"@webassemblyjs/helper-wasm-bytecode@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz#f328241e41e7b199d0b20c18e88429c4433295e1" + integrity sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q== -"@webassemblyjs/helper-wasm-section@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.9.0.tgz" - integrity sha512-XnMB8l3ek4tvrKUUku+IVaXNHz2YsJyOOmz+MMkZvh8h1uSJpSen6vYnw3IoQ7WwEuAhL8Efjms1ZWjqh2agvw== +"@webassemblyjs/helper-wasm-section@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz#21ee065a7b635f319e738f0dd73bfbda281c097a" + integrity sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg== dependencies: - "@webassemblyjs/ast" "1.9.0" - "@webassemblyjs/helper-buffer" "1.9.0" - "@webassemblyjs/helper-wasm-bytecode" "1.9.0" - "@webassemblyjs/wasm-gen" "1.9.0" + "@webassemblyjs/ast" "1.11.1" + "@webassemblyjs/helper-buffer" "1.11.1" + "@webassemblyjs/helper-wasm-bytecode" "1.11.1" + "@webassemblyjs/wasm-gen" "1.11.1" -"@webassemblyjs/ieee754@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.9.0.tgz" - integrity sha512-dcX8JuYU/gvymzIHc9DgxTzUUTLexWwt8uCTWP3otys596io0L5aW02Gb1RjYpx2+0Jus1h4ZFqjla7umFniTg== +"@webassemblyjs/ieee754@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz#963929e9bbd05709e7e12243a099180812992614" + integrity sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ== dependencies: "@xtuc/ieee754" "^1.2.0" -"@webassemblyjs/leb128@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.9.0.tgz" - integrity sha512-ENVzM5VwV1ojs9jam6vPys97B/S65YQtv/aanqnU7D8aSoHFX8GyhGg0CMfyKNIHBuAVjy3tlzd5QMMINa7wpw== +"@webassemblyjs/leb128@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/leb128/-/leb128-1.11.1.tgz#ce814b45574e93d76bae1fb2644ab9cdd9527aa5" + integrity sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw== dependencies: "@xtuc/long" "4.2.2" -"@webassemblyjs/utf8@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.9.0.tgz" - integrity sha512-GZbQlWtopBTP0u7cHrEx+73yZKrQoBMpwkGEIqlacljhXCkVM1kMQge/Mf+csMJAjEdSwhOyLAS0AoR3AG5P8w== +"@webassemblyjs/utf8@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/utf8/-/utf8-1.11.1.tgz#d1f8b764369e7c6e6bae350e854dec9a59f0a3ff" + integrity sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ== -"@webassemblyjs/wasm-edit@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.9.0.tgz" - integrity sha512-FgHzBm80uwz5M8WKnMTn6j/sVbqilPdQXTWraSjBwFXSYGirpkSWE2R9Qvz9tNiTKQvoKILpCuTjBKzOIm0nxw== - dependencies: - "@webassemblyjs/ast" "1.9.0" - "@webassemblyjs/helper-buffer" "1.9.0" - "@webassemblyjs/helper-wasm-bytecode" "1.9.0" - "@webassemblyjs/helper-wasm-section" "1.9.0" - "@webassemblyjs/wasm-gen" "1.9.0" - "@webassemblyjs/wasm-opt" "1.9.0" - "@webassemblyjs/wasm-parser" "1.9.0" - "@webassemblyjs/wast-printer" "1.9.0" - -"@webassemblyjs/wasm-gen@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.9.0.tgz" - integrity sha512-cPE3o44YzOOHvlsb4+E9qSqjc9Qf9Na1OO/BHFy4OI91XDE14MjFN4lTMezzaIWdPqHnsTodGGNP+iRSYfGkjA== +"@webassemblyjs/wasm-edit@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz#ad206ebf4bf95a058ce9880a8c092c5dec8193d6" + integrity sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA== + dependencies: + "@webassemblyjs/ast" "1.11.1" + "@webassemblyjs/helper-buffer" "1.11.1" + "@webassemblyjs/helper-wasm-bytecode" "1.11.1" + "@webassemblyjs/helper-wasm-section" "1.11.1" + "@webassemblyjs/wasm-gen" "1.11.1" + "@webassemblyjs/wasm-opt" "1.11.1" + "@webassemblyjs/wasm-parser" "1.11.1" + "@webassemblyjs/wast-printer" "1.11.1" + +"@webassemblyjs/wasm-gen@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz#86c5ea304849759b7d88c47a32f4f039ae3c8f76" + integrity sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA== dependencies: - "@webassemblyjs/ast" "1.9.0" - "@webassemblyjs/helper-wasm-bytecode" "1.9.0" - "@webassemblyjs/ieee754" "1.9.0" - "@webassemblyjs/leb128" "1.9.0" - "@webassemblyjs/utf8" "1.9.0" + "@webassemblyjs/ast" "1.11.1" + "@webassemblyjs/helper-wasm-bytecode" "1.11.1" + "@webassemblyjs/ieee754" "1.11.1" + "@webassemblyjs/leb128" "1.11.1" + "@webassemblyjs/utf8" "1.11.1" -"@webassemblyjs/wasm-opt@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.9.0.tgz" - integrity sha512-Qkjgm6Anhm+OMbIL0iokO7meajkzQD71ioelnfPEj6r4eOFuqm4YC3VBPqXjFyyNwowzbMD+hizmprP/Fwkl2A== +"@webassemblyjs/wasm-opt@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz#657b4c2202f4cf3b345f8a4c6461c8c2418985f2" + integrity sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw== dependencies: - "@webassemblyjs/ast" "1.9.0" - "@webassemblyjs/helper-buffer" "1.9.0" - "@webassemblyjs/wasm-gen" "1.9.0" - "@webassemblyjs/wasm-parser" "1.9.0" + "@webassemblyjs/ast" "1.11.1" + "@webassemblyjs/helper-buffer" "1.11.1" + "@webassemblyjs/wasm-gen" "1.11.1" + "@webassemblyjs/wasm-parser" "1.11.1" -"@webassemblyjs/wasm-parser@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.9.0.tgz" - integrity sha512-9+wkMowR2AmdSWQzsPEjFU7njh8HTO5MqO8vjwEHuM+AMHioNqSBONRdr0NQQ3dVQrzp0s8lTcYqzUdb7YgELA== +"@webassemblyjs/wasm-parser@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz#86ca734534f417e9bd3c67c7a1c75d8be41fb199" + integrity sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA== dependencies: - "@webassemblyjs/ast" "1.9.0" - "@webassemblyjs/helper-api-error" "1.9.0" - "@webassemblyjs/helper-wasm-bytecode" "1.9.0" - "@webassemblyjs/ieee754" "1.9.0" - "@webassemblyjs/leb128" "1.9.0" - "@webassemblyjs/utf8" "1.9.0" + "@webassemblyjs/ast" "1.11.1" + "@webassemblyjs/helper-api-error" "1.11.1" + "@webassemblyjs/helper-wasm-bytecode" "1.11.1" + "@webassemblyjs/ieee754" "1.11.1" + "@webassemblyjs/leb128" "1.11.1" + "@webassemblyjs/utf8" "1.11.1" -"@webassemblyjs/wast-parser@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/wast-parser/-/wast-parser-1.9.0.tgz" - integrity sha512-qsqSAP3QQ3LyZjNC/0jBJ/ToSxfYJ8kYyuiGvtn/8MK89VrNEfwj7BPQzJVHi0jGTRK2dGdJ5PRqhtjzoww+bw== - dependencies: - "@webassemblyjs/ast" "1.9.0" - "@webassemblyjs/floating-point-hex-parser" "1.9.0" - "@webassemblyjs/helper-api-error" "1.9.0" - "@webassemblyjs/helper-code-frame" "1.9.0" - "@webassemblyjs/helper-fsm" "1.9.0" - "@xtuc/long" "4.2.2" - -"@webassemblyjs/wast-printer@1.9.0": - version "1.9.0" - resolved "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.9.0.tgz" - integrity sha512-2J0nE95rHXHyQ24cWjMKJ1tqB/ds8z/cyeOZxJhcb+rW+SQASVjuznUSmdz5GpVJTzU8JkhYut0D3siFDD6wsA== +"@webassemblyjs/wast-printer@1.11.1": + version "1.11.1" + resolved "https://registry.yarnpkg.com/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz#d0c73beda8eec5426f10ae8ef55cee5e7084c2f0" + integrity sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg== dependencies: - "@webassemblyjs/ast" "1.9.0" - "@webassemblyjs/wast-parser" "1.9.0" + "@webassemblyjs/ast" "1.11.1" "@xtuc/long" "4.2.2" "@webpack-cli/info@^1.1.0": @@ -782,12 +770,12 @@ "@xtuc/ieee754@^1.2.0": version "1.2.0" - resolved "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz" + resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790" integrity sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA== "@xtuc/long@4.2.2": version "4.2.2" - resolved "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz" + resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d" integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ== abab@^2.0.5: @@ -803,12 +791,22 @@ accepts@~1.3.8: mime-types "~2.1.34" negotiator "0.6.3" +acorn-import-assertions@^1.7.6: + version "1.8.0" + resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz#ba2b5939ce62c238db6d93d81c9b111b29b855e9" + integrity sha512-m7VZ3jwz4eK6A4Vtt8Ew1/mNbP24u0FhdyfA7fSvnJR6LMdfOYnmuIrrJAgrYfYJ10F/otaHTtrtrtmHdMNzEw== + acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== -acorn@^8.0.4, acorn@^8.5.0, acorn@^8.8.0: +acorn@^8.5.0, acorn@^8.7.1: + version "8.8.2" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" + integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== + +acorn@^8.8.0: version "8.8.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.1.tgz#0a3f9cbecc4ec3bea6f0a80b66ae8dd2da250b73" integrity sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA== @@ -1027,15 +1025,14 @@ browser-stdout@1.3.1: integrity sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw== browserslist@^4.14.5: - version "4.16.6" - resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.16.6.tgz" - integrity sha512-Wspk/PqO+4W9qp5iUTJsa1B/QrYn1keNCcEP5OvP7WBwT4KaDly0uONYmC6Xa3Z5IqnUgS0KcgLYu1l74x0ZXQ== + version "4.21.5" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.5.tgz#75c5dae60063ee641f977e00edd3cfb2fb7af6a7" + integrity sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w== dependencies: - caniuse-lite "^1.0.30001219" - colorette "^1.2.2" - electron-to-chromium "^1.3.723" - escalade "^3.1.1" - node-releases "^1.1.71" + caniuse-lite "^1.0.30001449" + electron-to-chromium "^1.4.284" + node-releases "^2.0.8" + update-browserslist-db "^1.0.10" browserslist@^4.21.3, browserslist@^4.21.4: version "4.21.4" @@ -1088,16 +1085,16 @@ camelcase@^6.0.0: resolved "https://registry.npmjs.org/camelcase/-/camelcase-6.2.0.tgz" integrity sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg== -caniuse-lite@^1.0.30001219: - version "1.0.30001230" - resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001230.tgz" - integrity sha512-5yBd5nWCBS+jWKTcHOzXwo5xzcj4ePE/yjtkZyUV1BTUmrBaA9MRGC+e7mxnqXSA90CmCA8L3eKLaSUkt099IQ== - caniuse-lite@^1.0.30001400: version "1.0.30001439" resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001439.tgz#ab7371faeb4adff4b74dad1718a6fd122e45d9cb" integrity sha512-1MgUzEkoMO6gKfXflStpYgZDlFM7M/ck/bgfVCACO5vnAf0fXoNVHdWtqGU+MYca+4bL9Z5bpOVmR33cWW9G2A== +caniuse-lite@^1.0.30001449: + version "1.0.30001457" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001457.tgz#6af34bb5d720074e2099432aa522c21555a18301" + integrity sha512-SDIV6bgE1aVbK6XyxdURbUE89zY7+k1BBBaOwYwkNCglXlel/E7mELiHC64HQ+W0xSKlqWhV9Wh7iHxUjMs4fA== + chai@^4.2.0: version "4.2.0" resolved "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz" @@ -1156,11 +1153,9 @@ chokidar@3.5.3: fsevents "~2.3.2" chrome-trace-event@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.2.tgz" - integrity sha512-9e/zx1jw7B4CO+c/RXoCsfg/x1AfUBioy4owYH0bJprEYAx5hRFLRhWBqHAG57D0ZM4H7vxbP7bPe0VwhQRYDQ== - dependencies: - tslib "^1.9.0" + version "1.0.3" + resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz#1015eced4741e15d06664a957dbbf50d041e26ac" + integrity sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg== clipboard-copy@^4.0.1: version "4.0.1" @@ -1210,7 +1205,7 @@ color-name@~1.1.4: resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== -colorette@^1.2.1, colorette@^1.2.2: +colorette@^1.2.1: version "1.2.2" resolved "https://registry.npmjs.org/colorette/-/colorette-1.2.2.tgz" integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w== @@ -1420,16 +1415,16 @@ ee-first@1.1.1: resolved "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= -electron-to-chromium@^1.3.723: - version "1.3.739" - resolved "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.739.tgz" - integrity sha512-+LPJVRsN7hGZ9EIUUiWCpO7l4E3qBYHNadazlucBfsXBbccDFNKUBAgzE68FnkWGJPwD/AfKhSzL+G+Iqb8A4A== - electron-to-chromium@^1.4.251: version "1.4.284" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz#61046d1e4cab3a25238f6bf7413795270f125592" integrity sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA== +electron-to-chromium@^1.4.284: + version "1.4.302" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.302.tgz#5770646ffe7051677b489226144aad9386d420f2" + integrity sha512-Uk7C+7aPBryUR1Fwvk9VmipBcN9fVsqBO57jV2ZjTm+IZ6BMNqu7EDVEg2HxCNufk6QcWlFsBkhQyQroB2VWKw== + emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" @@ -1452,13 +1447,13 @@ end-of-stream@^1.1.0: dependencies: once "^1.4.0" -enhanced-resolve@^5.3.1: - version "5.3.2" - resolved "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.3.2.tgz" - integrity sha512-G28GCrglCAH6+EqMN2D+Q2wCUS1O1vVQJBn8ME2I/Api41YBe4vLWWRBOUbwDH7vwzSZdljxwTRVqnf+sm6XqQ== +enhanced-resolve@^5.10.0: + version "5.12.0" + resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.12.0.tgz#300e1c90228f5b570c4d35babf263f6da7155634" + integrity sha512-QHTXI/sZQmko1cbDoNAa3mJ5qhWUUNAq3vR0/YiD379fWQrcfuoX1+HW2S0MTt7XmoPLapdaDKUtelUSPic7hQ== dependencies: graceful-fs "^4.2.4" - tapable "^2.0.0" + tapable "^2.2.0" enquirer@^2.3.6: version "2.3.6" @@ -1524,6 +1519,11 @@ es-abstract@^1.19.0, es-abstract@^1.20.4: string.prototype.trimstart "^1.0.5" unbox-primitive "^1.0.2" +es-module-lexer@^0.9.0: + version "0.9.3" + resolved "https://registry.yarnpkg.com/es-module-lexer/-/es-module-lexer-0.9.3.tgz#6f13db00cc38417137daf74366f535c8eb438f19" + integrity sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ== + es-shim-unscopables@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/es-shim-unscopables/-/es-shim-unscopables-1.0.0.tgz#702e632193201e3edf8713635d083d378e510241" @@ -1581,7 +1581,7 @@ eslint-plugin-react@^7.31.11: semver "^6.3.0" string.prototype.matchall "^4.0.8" -eslint-scope@^5.1.1: +eslint-scope@5.1.1, eslint-scope@^5.1.1: version "5.1.1" resolved "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz" integrity sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw== @@ -1713,9 +1713,9 @@ etag@~1.8.1: integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= events@^3.2.0: - version "3.2.0" - resolved "https://registry.npmjs.org/events/-/events-3.2.0.tgz" - integrity sha512-/46HWwbfCX2xTawVfkKLGxMifJYQBWMwY1mjywRtb4c9x8l5NP3KoJtnIOiL1hfdRkIuYhETxQlo62IF8tcnlg== + version "3.3.0" + resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" + integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== execa@^4.1.0: version "4.1.0" @@ -1973,7 +1973,7 @@ glob-parent@^6.0.2: glob-to-regexp@^0.4.1: version "0.4.1" - resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz" + resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz#c75297087c851b9a578bd217dd59a92f59fe546e" integrity sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw== glob@7.2.0: @@ -2019,15 +2019,10 @@ globby@^11.1.0: merge2 "^1.4.1" slash "^3.0.0" -graceful-fs@^4.1.2: - version "4.2.3" - resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz" - integrity sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ== - -graceful-fs@^4.2.4: - version "4.2.4" - resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz" - integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== +graceful-fs@^4.1.2, graceful-fs@^4.2.4, graceful-fs@^4.2.9: + version "4.2.10" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" + integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== grapheme-splitter@^1.0.4: version "1.0.4" @@ -2397,14 +2392,14 @@ isexe@^2.0.0: resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= -jest-worker@^26.6.1: - version "26.6.2" - resolved "https://registry.npmjs.org/jest-worker/-/jest-worker-26.6.2.tgz" - integrity sha512-KWYVV1c4i+jbMpaBC+U++4Va0cp8OisU185o73T1vo99hqi7w8tSJfUXYswwqqrjzwxa6KpRK54WhPvwf5w6PQ== +jest-worker@^27.4.5: + version "27.5.1" + resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-27.5.1.tgz#8d146f0900e8973b106b6f73cc1e9a8cb86f8db0" + integrity sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg== dependencies: "@types/node" "*" merge-stream "^2.0.0" - supports-color "^7.0.0" + supports-color "^8.0.0" js-sdsl@^4.1.4: version "4.2.0" @@ -2423,10 +2418,10 @@ js-yaml@4.1.0, js-yaml@^4.1.0: dependencies: argparse "^2.0.1" -json-parse-better-errors@^1.0.2: - version "1.0.2" - resolved "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz" - integrity sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw== +json-parse-even-better-errors@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" + integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== json-schema-traverse@^0.4.1: version "0.4.1" @@ -2547,10 +2542,10 @@ linkify-it@^3.0.1: dependencies: uc.micro "^1.0.1" -loader-runner@^4.1.0: - version "4.1.0" - resolved "https://registry.npmjs.org/loader-runner/-/loader-runner-4.1.0.tgz" - integrity sha512-oR4lB4WvwFoC70ocraKhn5nkKSs23t57h9udUgw8o0iH8hMXeEoRuUgfcvgUwAJ1ZpRqBvcou4N2SMvM1DwMrA== +loader-runner@^4.2.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.3.0.tgz#c1b4a163b99f614830353b16755e7149ac2314e1" + integrity sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg== loader-utils@^2.0.0: version "2.0.4" @@ -2681,20 +2676,20 @@ mime-db@1.52.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.1.27, mime-types@~2.1.24: - version "2.1.27" - resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz" - integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w== - dependencies: - mime-db "1.44.0" - -mime-types@~2.1.34: +mime-types@^2.1.27, mime-types@~2.1.34: version "2.1.35" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== dependencies: mime-db "1.52.0" +mime-types@~2.1.24: + version "2.1.27" + resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz" + integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w== + dependencies: + mime-db "1.44.0" + mime@1.6.0: version "1.6.0" resolved "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz" @@ -2808,19 +2803,19 @@ negotiator@0.6.3: neo-async@^2.6.2: version "2.6.2" - resolved "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz" + resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== -node-releases@^1.1.71: - version "1.1.72" - resolved "https://registry.npmjs.org/node-releases/-/node-releases-1.1.72.tgz" - integrity sha512-LLUo+PpH3dU6XizX3iVoubUNheF/owjXCZZ5yACDxNnPtgFuludV1ZL3ayK1kVep42Rmm0+R9/Y60NQbZ2bifw== - node-releases@^2.0.6: version "2.0.8" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.8.tgz#0f349cdc8fcfa39a92ac0be9bc48b7706292b9ae" integrity sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A== +node-releases@^2.0.8: + version "2.0.10" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.10.tgz#c311ebae3b6a148c89b1813fd7c4d3c024ef537f" + integrity sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w== + normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz" @@ -3230,6 +3225,21 @@ react-dropzone@^14.2.1: file-selector "^0.6.0" prop-types "^15.8.1" +react-fast-compare@^3.1.1: + version "3.2.0" + resolved "https://registry.yarnpkg.com/react-fast-compare/-/react-fast-compare-3.2.0.tgz#641a9da81b6a6320f270e89724fb45a0b39e43bb" + integrity sha512-rtGImPZ0YyLrscKI9xTpV8psd6I8VAtjKCzQDlzyDvqJA8XOW78TXYQwNRNd8g8JZnDu8q9Fu/1v4HPAVwVdHA== + +react-helmet@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/react-helmet/-/react-helmet-6.1.0.tgz#a750d5165cb13cf213e44747502652e794468726" + integrity sha512-4uMzEY9nlDlgxr61NL3XbKRy1hEkXmKNXhjbAIOVw5vcFrsdYbH2FEwcNyWvWinl103nXgzYNlns9ca+8kFiWw== + dependencies: + object-assign "^4.1.1" + prop-types "^15.7.2" + react-fast-compare "^3.1.1" + react-side-effect "^2.1.0" + react-infinite-scroller@^1.2.4: version "1.2.4" resolved "https://registry.npmjs.org/react-infinite-scroller/-/react-infinite-scroller-1.2.4.tgz" @@ -3271,6 +3281,11 @@ react-router@5.2.0: tiny-invariant "^1.0.2" tiny-warning "^1.0.0" +react-side-effect@^2.1.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/react-side-effect/-/react-side-effect-2.1.2.tgz#dc6345b9e8f9906dc2eeb68700b615e0b4fe752a" + integrity sha512-PVjOcvVOyIILrYoyGEpDN3vmYNLdy1CajSFNt4TDsVQC5KpTijDvWVoR+/7Rz2xT978D8/ZtFceXxzsPwZEDvw== + react-transition-group@^4.4.0: version "4.4.1" resolved "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.1.tgz" @@ -3444,6 +3459,15 @@ schema-utils@^3.0.0: ajv "^6.12.5" ajv-keywords "^3.5.2" +schema-utils@^3.1.0, schema-utils@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.1.1.tgz#bc74c4b6b6995c1d88f76a8b77bea7219e0c8281" + integrity sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw== + dependencies: + "@types/json-schema" "^7.0.8" + ajv "^6.12.5" + ajv-keywords "^3.5.2" + semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" @@ -3482,10 +3506,10 @@ serialize-javascript@6.0.0: dependencies: randombytes "^2.1.0" -serialize-javascript@^5.0.1: - version "5.0.1" - resolved "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz" - integrity sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA== +serialize-javascript@^6.0.0: + version "6.0.1" + resolved "https://registry.yarnpkg.com/serialize-javascript/-/serialize-javascript-6.0.1.tgz#b206efb27c3da0b0ab6b52f48d170b7996458e5c" + integrity sha512-owoXEFjWRllis8/M1Q+Cw5k8ZH40e3zhp/ovX+Xr/vi1qj6QesbyXXViFbpNvWvPNAD62SutwEXavefrLJWj7w== dependencies: randombytes "^2.1.0" @@ -3535,11 +3559,6 @@ slash@^3.0.0: resolved "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz" integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== -source-list-map@^2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz" - integrity sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw== - source-map-js@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" @@ -3662,7 +3681,7 @@ style-loader@^2.0.0: loader-utils "^2.0.0" schema-utils "^3.0.0" -supports-color@8.1.1: +supports-color@8.1.1, supports-color@^8.0.0: version "8.1.1" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== @@ -3676,13 +3695,6 @@ supports-color@^5.3.0: dependencies: has-flag "^3.0.0" -supports-color@^7.0.0: - version "7.2.0" - resolved "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz" - integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== - dependencies: - has-flag "^4.0.0" - supports-color@^7.1.0: version "7.1.0" resolved "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz" @@ -3705,27 +3717,26 @@ table-layout@^1.0.1: typical "^5.2.0" wordwrapjs "^4.0.0" -tapable@^2.0.0, tapable@^2.1.1: - version "2.1.1" - resolved "https://registry.npmjs.org/tapable/-/tapable-2.1.1.tgz" - integrity sha512-Wib1S8m2wdpLbmQz0RBEVosIyvb/ykfKXf3ZIDqvWoMg/zTNm6G/tDSuUM61J1kNCDXWJrLHGSFeMhAG+gAGpQ== +tapable@^2.1.1, tapable@^2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" + integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== -terser-webpack-plugin@^5.0.3: - version "5.0.3" - resolved "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.0.3.tgz" - integrity sha512-zFdGk8Lh9ZJGPxxPE6jwysOlATWB8GMW8HcfGULWA/nPal+3VdATflQvSBSLQJRCmYZnfFJl6vkRTiwJGNgPiQ== +terser-webpack-plugin@^5.1.3: + version "5.3.6" + resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.6.tgz#5590aec31aa3c6f771ce1b1acca60639eab3195c" + integrity sha512-kfLFk+PoLUQIbLmB1+PZDMRSZS99Mp+/MHqDNmMA6tOItzRt+Npe3E+fsMs5mfcM0wCtrrdU387UnV+vnSffXQ== dependencies: - jest-worker "^26.6.1" - p-limit "^3.0.2" - schema-utils "^3.0.0" - serialize-javascript "^5.0.1" - source-map "^0.6.1" - terser "^5.3.8" + "@jridgewell/trace-mapping" "^0.3.14" + jest-worker "^27.4.5" + schema-utils "^3.1.1" + serialize-javascript "^6.0.0" + terser "^5.14.1" -terser@^5.3.8: - version "5.15.1" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.15.1.tgz#8561af6e0fd6d839669c73b92bdd5777d870ed6c" - integrity sha512-K1faMUvpm/FBxjBXud0LWVAGxmvoPbZbfTCYbSgaaYQaIXI3/TdI7a7ZGA73Zrou6Q8Zmz3oeUTsp/dj+ag2Xw== +terser@^5.14.1: + version "5.16.4" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.16.4.tgz#51284b440b93242291a98f2a9903c024cfb70e6e" + integrity sha512-5yEGuZ3DZradbogeYQ1NaGz7rXVBDWujWlx1PT8efXO6Txn+eWbfKqB2bTDVmFXmePFkoLU6XI8UektMIEA0ug== dependencies: "@jridgewell/source-map" "^0.3.2" acorn "^8.5.0" @@ -3797,7 +3808,7 @@ tsconfig-paths@^3.5.0: minimist "^1.2.0" strip-bom "^3.0.0" -tslib@^1.8.1, tslib@^1.9.0: +tslib@^1.8.1: version "1.11.1" resolved "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz" integrity sha512-aZW88SY8kQbU7gpV19lN24LtXh/yD4ZZg6qieAJDDg+YBsJcSmLGK9QpnUjAKVG/xefmvJGd1WUmfpT/g6AJGA== @@ -3874,7 +3885,7 @@ unpipe@1.0.0, unpipe@~1.0.0: resolved "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= -update-browserslist-db@^1.0.9: +update-browserslist-db@^1.0.10, update-browserslist-db@^1.0.9: version "1.0.10" resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz#0f54b876545726f17d00cd9a2561e6dade943ff3" integrity sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ== @@ -3914,10 +3925,10 @@ vary@~1.1.2: resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= -watchpack@^2.0.0: - version "2.0.1" - resolved "https://registry.npmjs.org/watchpack/-/watchpack-2.0.1.tgz" - integrity sha512-vO8AKGX22ZRo6PiOFM9dC0re8IcKh8Kd/aH2zeqUc6w4/jBGlTy2P7fTC6ekT0NjVeGjgU2dGC5rNstKkeLEQg== +watchpack@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.4.0.tgz#fa33032374962c78113f93c7f2fb4c54c9862a5d" + integrity sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg== dependencies: glob-to-regexp "^0.4.1" graceful-fs "^4.1.2" @@ -3948,43 +3959,40 @@ webpack-merge@^4.2.2: dependencies: lodash "^4.17.15" -webpack-sources@^2.1.1: - version "2.2.0" - resolved "https://registry.npmjs.org/webpack-sources/-/webpack-sources-2.2.0.tgz" - integrity sha512-bQsA24JLwcnWGArOKUxYKhX3Mz/nK1Xf6hxullKERyktjNMC4x8koOeaDNTA2fEJ09BdWLbM/iTW0ithREUP0w== - dependencies: - source-list-map "^2.0.1" - source-map "^0.6.1" - -webpack@^5.9.0: - version "5.9.0" - resolved "https://registry.npmjs.org/webpack/-/webpack-5.9.0.tgz" - integrity sha512-YnnqIV/uAS5ZrNpctSv378qV7HmbJ74DL+XfvMxzbX1bV9e7eeT6eEWU4wuUw33CNr/HspBh7R/xQlVjTEyAeA== - dependencies: - "@types/eslint-scope" "^3.7.0" - "@types/estree" "^0.0.45" - "@webassemblyjs/ast" "1.9.0" - "@webassemblyjs/helper-module-context" "1.9.0" - "@webassemblyjs/wasm-edit" "1.9.0" - "@webassemblyjs/wasm-parser" "1.9.0" - acorn "^8.0.4" +webpack-sources@^3.2.3: + version "3.2.3" + resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-3.2.3.tgz#2d4daab8451fd4b240cc27055ff6a0c2ccea0cde" + integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== + +webpack@^5.75.0: + version "5.75.0" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.75.0.tgz#1e440468647b2505860e94c9ff3e44d5b582c152" + integrity sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ== + dependencies: + "@types/eslint-scope" "^3.7.3" + "@types/estree" "^0.0.51" + "@webassemblyjs/ast" "1.11.1" + "@webassemblyjs/wasm-edit" "1.11.1" + "@webassemblyjs/wasm-parser" "1.11.1" + acorn "^8.7.1" + acorn-import-assertions "^1.7.6" browserslist "^4.14.5" chrome-trace-event "^1.0.2" - enhanced-resolve "^5.3.1" - eslint-scope "^5.1.1" + enhanced-resolve "^5.10.0" + es-module-lexer "^0.9.0" + eslint-scope "5.1.1" events "^3.2.0" glob-to-regexp "^0.4.1" - graceful-fs "^4.2.4" - json-parse-better-errors "^1.0.2" - loader-runner "^4.1.0" + graceful-fs "^4.2.9" + json-parse-even-better-errors "^2.3.1" + loader-runner "^4.2.0" mime-types "^2.1.27" neo-async "^2.6.2" - pkg-dir "^4.2.0" - schema-utils "^3.0.0" + schema-utils "^3.1.0" tapable "^2.1.1" - terser-webpack-plugin "^5.0.3" - watchpack "^2.0.0" - webpack-sources "^2.1.1" + terser-webpack-plugin "^5.1.3" + watchpack "^2.4.0" + webpack-sources "^3.2.3" whatwg-mimetype@^2.3.0: version "2.3.0" From 13594aa2db96518d5ee6c28280523f26e2c598ff Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Thu, 23 Feb 2023 11:48:09 +0200 Subject: [PATCH 14/45] export NamespaceDetailComponent bump webui version # Conflicts: # webui/package.json --- webui/package.json | 2 +- webui/src/index.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/webui/package.json b/webui/package.json index 40dbe7d90..310e86eac 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "openvsx-webui", - "version": "0.9.0", + "version": "0.9.1", "description": "User interface for Eclipse Open VSX", "keywords": [ "react", diff --git a/webui/src/index.ts b/webui/src/index.ts index 142ddd119..1b754e9e3 100644 --- a/webui/src/index.ts +++ b/webui/src/index.ts @@ -14,4 +14,5 @@ export * from './extension-registry-service'; export * from './extension-registry-types'; export * from './pages/extension-detail/extension-detail'; export * from './pages/extension-list/extension-list'; +export * from './pages/namespace-detail/namespace-detail'; export * from './pages/user/user-settings'; From 8845246c86872fcf9eb57ae2ceb6a3550eb272fa Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Thu, 23 Feb 2023 14:00:37 +0200 Subject: [PATCH 15/45] Move mainHeadTags to main.tsx --- webui/package.json | 2 +- webui/src/default/default-app.tsx | 2 -- webui/src/main.tsx | 4 +++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/webui/package.json b/webui/package.json index 310e86eac..95d15c939 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "openvsx-webui", - "version": "0.9.1", + "version": "0.9.2", "description": "User interface for Eclipse Open VSX", "keywords": [ "react", diff --git a/webui/src/default/default-app.tsx b/webui/src/default/default-app.tsx index bb9cb30ab..21780f5cd 100644 --- a/webui/src/default/default-app.tsx +++ b/webui/src/default/default-app.tsx @@ -40,9 +40,7 @@ const App = () => { ); const pageSettings = createPageSettings(theme, prefersDarkMode, service.serverUrl); - const { mainHeadTags: MainHeadTagsComponent } = pageSettings.elements; return (<> - { MainHeadTagsComponent ? : null}
+ { MainHeadTagsComponent ? : null } From 24336c74424bd02a5bcc47f00e308845afbd88d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Tron=C3=AD=C4=8Dek?= Date: Thu, 23 Feb 2023 22:24:31 +0000 Subject: [PATCH 16/45] Update test extensions --- server/test-extensions.gradle | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/server/test-extensions.gradle b/server/test-extensions.gradle index 09301a969..9dc482d5a 100644 --- a/server/test-extensions.gradle +++ b/server/test-extensions.gradle @@ -29,24 +29,28 @@ task downloadTestExtensions { dest "${buildDir}/test-extensions/ms-python.python-2019.10.44104.vsix" } download { - src "https://github.com/redhat-developer/vscode-java/releases/download/v1.11.0/redhat.java@darwin-arm64-1.11.0.vsix" - dest "${buildDir}/test-extensions/redhat.java@darwin-arm64-1.11.0.vsix" + src "https://github.com/redhat-developer/vscode-java/releases/download/v1.15.0/redhat.java@darwin-arm64-1.15.0.vsix" + dest "${buildDir}/test-extensions/redhat.java@darwin-arm64-1.15.0.vsix" } download { - src "https://github.com/redhat-developer/vscode-java/releases/download/v1.11.0/redhat.java@darwin-x64-1.11.0.vsix" - dest "${buildDir}/test-extensions/redhat.java@darwin-x64-1.11.0.vsix" + src "https://github.com/redhat-developer/vscode-java/releases/download/v1.15.0/redhat.java@darwin-x64-1.15.0.vsix" + dest "${buildDir}/test-extensions/redhat.java@darwin-x64-1.15.0.vsix" } download { - src "https://github.com/redhat-developer/vscode-java/releases/download/v1.11.0/redhat.java@linux-arm64-1.11.0.vsix" - dest "${buildDir}/test-extensions/redhat.java@linux-arm64-1.11.0.vsix" + src "https://github.com/redhat-developer/vscode-java/releases/download/v1.15.0/redhat.java@linux-arm64-1.15.0.vsix" + dest "${buildDir}/test-extensions/redhat.java@linux-arm64-1.15.0.vsix" } download { - src "https://github.com/redhat-developer/vscode-java/releases/download/v1.11.0/redhat.java@linux-x64-1.11.0.vsix" - dest "${buildDir}/test-extensions/redhat.java@linux-x64-1.11.0.vsix" + src "https://github.com/redhat-developer/vscode-java/releases/download/v1.15.0/redhat.java@linux-x64-1.15.0.vsix" + dest "${buildDir}/test-extensions/redhat.java@linux-x64-1.15.0.vsix" } download { - src "https://github.com/redhat-developer/vscode-java/releases/download/v1.11.0/redhat.java@win32-x64-1.11.0.vsix" - dest "${buildDir}/test-extensions/redhat.java@win32-x64-1.11.0.vsix" + src "https://github.com/redhat-developer/vscode-java/releases/download/v1.15.0/redhat.java@win32-x64-1.15.0.vsix" + dest "${buildDir}/test-extensions/redhat.java@win32-x64-1.15.0.vsix" + } + download { + src "https://github.com/redhat-developer/vscode-java/releases/download/v1.15.0/java-1.15.0.vsix" + dest "${buildDir}/test-extensions/redhat.java@win32-x64-1.15.0.vsix" } download { src "https://github.com/HookyQR/VSCodeBeautify/releases/download/v0.1.3/beautify-0.1.3.vsix" From 7ff8e3012e9335bd492c13099986250388afca32 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Fri, 24 Feb 2023 10:04:55 +0200 Subject: [PATCH 17/45] Merge entity before calling remove --- .../java/org/eclipse/openvsx/migration/MigrationService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java b/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java index b5973d1e9..8c2cbda29 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java @@ -124,6 +124,7 @@ public void persistFileResource(FileResource resource) { @Transactional public void deleteFileResource(FileResource resource) { + resource = entityManager.merge(resource); entityManager.remove(resource); } From 2e7e8e14d5c2be7c9b7a4c6d1ab2961b00084f47 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Sat, 25 Feb 2023 12:13:03 +0200 Subject: [PATCH 18/45] Read input stream before try to get VSIX manifest from zip file --- server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java b/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java index b412c42d5..2a8bc5826 100644 --- a/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java +++ b/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java @@ -469,6 +469,7 @@ protected FileResource getIcon(ExtensionVersion extVersion) { } public FileResource getVsixManifest(ExtensionVersion extVersion) { + readInputStream(); var vsixManifest = new FileResource(); vsixManifest.setExtension(extVersion); vsixManifest.setName(VSIX_MANIFEST); From 7a16ecb919134c01dc8a450945c970052d4ede14 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Tue, 28 Feb 2023 12:03:08 +0200 Subject: [PATCH 19/45] Use explicit joins to improve query performance --- .../FileResourceJooqRepository.java | 37 ++++++++++++++++--- .../repositories/FileResourceRepository.java | 2 - .../repositories/RepositoryService.java | 4 +- .../org/eclipse/openvsx/AdminAPITest.java | 7 +--- .../org/eclipse/openvsx/RegistryAPITest.java | 6 +-- 5 files changed, 38 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java index 7f7cd7dc6..31ee602c0 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java @@ -18,8 +18,10 @@ import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; -import static org.eclipse.openvsx.jooq.Tables.*; +import static org.eclipse.openvsx.jooq.Tables.FILE_RESOURCE; @Component public class FileResourceJooqRepository { @@ -27,11 +29,25 @@ public class FileResourceJooqRepository { @Autowired DSLContext dsl; + public List findByType(Collection extVersions, Collection types) { + var extVersionsById = extVersions.stream().collect(Collectors.toMap(ExtensionVersion::getId, ev -> ev)); + var frExt = FILE_RESOURCE.as("fr_ext"); + var frType = FILE_RESOURCE.as("fr_type"); + return dsl.select(FILE_RESOURCE.ID, FILE_RESOURCE.EXTENSION_ID, FILE_RESOURCE.NAME, FILE_RESOURCE.TYPE) + .from(FILE_RESOURCE) + .join(frExt).on(frExt.EXTENSION_ID.in(extVersionsById.keySet()).and(frExt.ID.eq(FILE_RESOURCE.ID))) + .join(frType).on(frType.ID.eq(frExt.ID).and(frType.TYPE.in(types))) + .fetch() + .map(record -> toFileResource(record, extVersionsById)); + } + public List findAll(Collection extensionIds, Collection types) { + var frExt = FILE_RESOURCE.as("fr_ext"); + var frType = FILE_RESOURCE.as("fr_type"); return dsl.select(FILE_RESOURCE.ID, FILE_RESOURCE.EXTENSION_ID, FILE_RESOURCE.NAME, FILE_RESOURCE.TYPE) .from(FILE_RESOURCE) - .where(FILE_RESOURCE.EXTENSION_ID.in(extensionIds)) - .and(FILE_RESOURCE.TYPE.in(types)) + .join(frExt).on(frExt.EXTENSION_ID.in(extensionIds).and(frExt.ID.eq(FILE_RESOURCE.ID))) + .join(frType).on(frType.ID.eq(frExt.ID).and(frType.TYPE.in(types))) .fetch() .map(this::toFileResource); } @@ -53,13 +69,22 @@ public List findAllResources(long extVersionId, String prefix) { } private FileResource toFileResource(Record record) { + var extVersion = new ExtensionVersion(); + extVersion.setId(record.get(FILE_RESOURCE.EXTENSION_ID)); + + return toFileResource(record, extVersion); + } + + private FileResource toFileResource(Record record, Map extVersionsById) { + var extVersion = extVersionsById.get(record.get(FILE_RESOURCE.EXTENSION_ID)); + return toFileResource(record, extVersion); + } + + private FileResource toFileResource(Record record, ExtensionVersion extVersion) { var fileResource = new FileResource(); fileResource.setId(record.get(FILE_RESOURCE.ID)); fileResource.setName(record.get(FILE_RESOURCE.NAME)); fileResource.setType(record.get(FILE_RESOURCE.TYPE)); - - var extVersion = new ExtensionVersion(); - extVersion.setId(record.get(FILE_RESOURCE.EXTENSION_ID)); fileResource.setExtension(extVersion); return fileResource; diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceRepository.java index efa122e2c..a0a19a4d7 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceRepository.java @@ -33,7 +33,5 @@ public interface FileResourceRepository extends Repository { Streamable findByExtensionAndTypeIn(ExtensionVersion extVersion, Collection types); - Streamable findByExtensionInAndTypeIn(Collection extVersions, Collection types); - void deleteByExtensionAndType(ExtensionVersion extVersion, String type); } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java index ec03f7b58..166967ad1 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java @@ -182,8 +182,8 @@ public Streamable findFilesByType(ExtensionVersion extVersion, Col return fileResourceRepo.findByExtensionAndTypeIn(extVersion, types); } - public Streamable findFilesByType(Collection extVersions, Collection types) { - return fileResourceRepo.findByExtensionInAndTypeIn(extVersions, types); + public List findFilesByType(Collection extVersions, Collection types) { + return fileResourceJooqRepo.findByType(extVersions, types); } public Streamable findActiveReviews(Extension extension) { diff --git a/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java b/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java index 5b8b6eda7..adfbe30c4 100644 --- a/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java @@ -48,10 +48,7 @@ import javax.persistence.EntityManager; import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -1383,7 +1380,7 @@ private List mockExtension(int numberOfVersions, int numberOfB Mockito.when(repositories.findFiles(extVersion)) .thenReturn(Streamable.empty()); Mockito.when(repositories.findFilesByType(anyCollection(), any())) - .thenReturn(Streamable.empty()); + .thenReturn(Collections.emptyList()); Mockito.when(repositories.findVersion(extVersion.getVersion(), TargetPlatform.NAME_UNIVERSAL, "baz", "foobar")) .thenReturn(extVersion); Mockito.when(repositories.findTargetPlatformVersions(extVersion.getVersion(), "baz", "foobar")) diff --git a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java index 79a735bf6..9628ca3c3 100644 --- a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java @@ -1844,8 +1844,8 @@ private ExtensionVersion mockExtension(String targetPlatform) { Collection extVersions = invocation.getArgument(0); Collection types = invocation.getArgument(1); return types.contains(DOWNLOAD) && extVersions.iterator().hasNext() && download.getExtension().equals(extVersions.iterator().next()) - ? Streamable.of(download) - : Streamable.empty(); + ? List.of(download) + : Collections.emptyList(); }); return extVersion; @@ -2028,7 +2028,7 @@ private void mockForPublish(String mode) { Mockito.when(repositories.findVersions(any(Extension.class))) .thenReturn(Streamable.empty()); Mockito.when(repositories.findFilesByType(anyCollection(), anyCollection())) - .thenReturn(Streamable.empty()); + .thenReturn(Collections.emptyList()); Mockito.when(repositories.findVersions(eq("1.0.0"), any(Extension.class))) .thenReturn(Streamable.empty()); if (mode.equals("owner")) { From 03fb490ec0c432fd45c3fa5a2cef9ff7ce4d8da3 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Tue, 28 Feb 2023 20:09:54 +0200 Subject: [PATCH 20/45] use in clauses --- .../repositories/FileResourceJooqRepository.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java index 31ee602c0..b30f5e3ea 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceJooqRepository.java @@ -31,23 +31,17 @@ public class FileResourceJooqRepository { public List findByType(Collection extVersions, Collection types) { var extVersionsById = extVersions.stream().collect(Collectors.toMap(ExtensionVersion::getId, ev -> ev)); - var frExt = FILE_RESOURCE.as("fr_ext"); - var frType = FILE_RESOURCE.as("fr_type"); return dsl.select(FILE_RESOURCE.ID, FILE_RESOURCE.EXTENSION_ID, FILE_RESOURCE.NAME, FILE_RESOURCE.TYPE) .from(FILE_RESOURCE) - .join(frExt).on(frExt.EXTENSION_ID.in(extVersionsById.keySet()).and(frExt.ID.eq(FILE_RESOURCE.ID))) - .join(frType).on(frType.ID.eq(frExt.ID).and(frType.TYPE.in(types))) + .where(FILE_RESOURCE.EXTENSION_ID.in(extVersionsById.keySet())).and(FILE_RESOURCE.TYPE.in(types)) .fetch() .map(record -> toFileResource(record, extVersionsById)); } public List findAll(Collection extensionIds, Collection types) { - var frExt = FILE_RESOURCE.as("fr_ext"); - var frType = FILE_RESOURCE.as("fr_type"); return dsl.select(FILE_RESOURCE.ID, FILE_RESOURCE.EXTENSION_ID, FILE_RESOURCE.NAME, FILE_RESOURCE.TYPE) .from(FILE_RESOURCE) - .join(frExt).on(frExt.EXTENSION_ID.in(extensionIds).and(frExt.ID.eq(FILE_RESOURCE.ID))) - .join(frType).on(frType.ID.eq(frExt.ID).and(frType.TYPE.in(types))) + .where(FILE_RESOURCE.EXTENSION_ID.in(extensionIds).and(FILE_RESOURCE.TYPE.in(types))) .fetch() .map(this::toFileResource); } From dd9735bd56a572ce092f13b6640106dcefb94435 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Wed, 1 Mar 2023 12:31:55 +0200 Subject: [PATCH 21/45] Delay running migrations Add possibility to delay running migrations, so that deployment can finish before migrations run --- .../openvsx/migration/HandlerJobRequest.java | 29 +++++++++++++++ .../openvsx/migration/MigrationRunner.java | 34 ++++++------------ .../openvsx/migration/MigrationScheduler.java | 35 +++++++++++++++++++ .../openvsx/migration/MigrationService.java | 17 +++++++++ 4 files changed, 91 insertions(+), 24 deletions(-) create mode 100644 server/src/main/java/org/eclipse/openvsx/migration/HandlerJobRequest.java create mode 100644 server/src/main/java/org/eclipse/openvsx/migration/MigrationScheduler.java diff --git a/server/src/main/java/org/eclipse/openvsx/migration/HandlerJobRequest.java b/server/src/main/java/org/eclipse/openvsx/migration/HandlerJobRequest.java new file mode 100644 index 000000000..871704018 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/migration/HandlerJobRequest.java @@ -0,0 +1,29 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.migration; + +import org.jobrunr.jobs.lambdas.JobRequest; +import org.jobrunr.jobs.lambdas.JobRequestHandler; + +public class HandlerJobRequest> implements JobRequest { + + private Class handler; + + public HandlerJobRequest() {} + + public HandlerJobRequest(Class handler) { + this.handler = handler; + } + + @Override + public Class getJobRequestHandler() { + return handler; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java b/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java index f9c18d0b9..fc898dfbe 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java @@ -9,21 +9,14 @@ * ****************************************************************************** */ package org.eclipse.openvsx.migration; -import org.eclipse.openvsx.entities.MigrationItem; import org.eclipse.openvsx.repositories.RepositoryService; +import org.jobrunr.jobs.annotations.Job; import org.jobrunr.jobs.lambdas.JobRequestHandler; -import org.jobrunr.scheduling.JobRequestScheduler; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.event.ApplicationStartedEvent; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; -import javax.transaction.Transactional; -import java.nio.charset.StandardCharsets; -import java.util.UUID; - @Component -public class MigrationRunner { +public class MigrationRunner implements JobRequestHandler> { @Autowired OrphanNamespaceMigration orphanNamespaceMigration; @@ -32,11 +25,11 @@ public class MigrationRunner { RepositoryService repositories; @Autowired - JobRequestScheduler scheduler; + MigrationService migrations; - @EventListener - @Transactional - public void runMigrations(ApplicationStartedEvent event) { + @Override + @Job(name = "Run migrations", retries = 0) + public void run(HandlerJobRequest jobRequest) throws Exception { orphanNamespaceMigration.fixOrphanNamespaces(); extractResourcesMigration(); setPreReleaseMigration(); @@ -47,31 +40,24 @@ public void runMigrations(ApplicationStartedEvent event) { private void extractResourcesMigration() { var jobName = "ExtractResourcesMigration"; var handler = ExtractResourcesJobRequestHandler.class; - repositories.findNotMigratedResources().forEach(item -> enqueueJob(jobName, handler, item)); + repositories.findNotMigratedResources().forEach(item -> migrations.enqueueMigration(jobName, handler, item)); } private void setPreReleaseMigration() { var jobName = "SetPreReleaseMigration"; var handler = SetPreReleaseJobRequestHandler.class; - repositories.findNotMigratedPreReleases().forEach(item -> enqueueJob(jobName, handler, item)); + repositories.findNotMigratedPreReleases().forEach(item -> migrations.enqueueMigration(jobName, handler, item)); } private void renameDownloadsMigration() { var jobName = "RenameDownloadsMigration"; var handler = RenameDownloadsJobRequestHandler.class; - repositories.findNotMigratedRenamedDownloads().forEach(item -> enqueueJob(jobName, handler, item)); + repositories.findNotMigratedRenamedDownloads().forEach(item -> migrations.enqueueMigration(jobName, handler, item)); } private void extractVsixManifestMigration() { var jobName = "ExtractVsixManifestMigration"; var handler = ExtractVsixManifestsJobRequestHandler.class; - repositories.findNotMigratedVsixManifests().forEach(item -> enqueueJob(jobName, handler, item)); - } - - private void enqueueJob(String jobName, Class> handler, MigrationItem item) { - var jobIdText = jobName + "::itemId=" + item.getId(); - var jobId = UUID.nameUUIDFromBytes(jobIdText.getBytes(StandardCharsets.UTF_8)); - scheduler.enqueue(jobId, new MigrationJobRequest<>(handler, item.getEntityId())); - item.setMigrationScheduled(true); + repositories.findNotMigratedVsixManifests().forEach(item -> migrations.enqueueMigration(jobName, handler, item)); } } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/MigrationScheduler.java b/server/src/main/java/org/eclipse/openvsx/migration/MigrationScheduler.java new file mode 100644 index 000000000..ef7e02575 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/migration/MigrationScheduler.java @@ -0,0 +1,35 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.migration; + +import org.jobrunr.scheduling.JobRequestScheduler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationStartedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.time.Instant; + +@Component +public class MigrationScheduler { + + @Autowired + JobRequestScheduler scheduler; + + @Value("${ovsx.migrations.delay.seconds:0}") + long delay; + + @EventListener + public void applicationStarted(ApplicationStartedEvent event) { + var instant = Instant.now().plusSeconds(delay); + scheduler.schedule(instant, new HandlerJobRequest<>(MigrationRunner.class)); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java b/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java index 8c2cbda29..3560d717f 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java @@ -11,10 +11,13 @@ import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.entities.MigrationItem; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.storage.AzureBlobStorageService; import org.eclipse.openvsx.storage.GoogleCloudStorageService; import org.eclipse.openvsx.storage.IStorageService; +import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.jobrunr.scheduling.JobRequestScheduler; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpMethod; import org.springframework.retry.annotation.Retryable; @@ -24,10 +27,12 @@ import javax.persistence.EntityManager; import javax.transaction.Transactional; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.AbstractMap; import java.util.Map; +import java.util.UUID; @Component public class MigrationService { @@ -47,6 +52,18 @@ public class MigrationService { @Autowired GoogleCloudStorageService googleStorage; + @Autowired + JobRequestScheduler scheduler; + + @Transactional + public void enqueueMigration(String jobName, Class> handler, MigrationItem item) { + item = entityManager.merge(item); + var jobIdText = jobName + "::itemId=" + item.getId(); + var jobId = UUID.nameUUIDFromBytes(jobIdText.getBytes(StandardCharsets.UTF_8)); + scheduler.enqueue(jobId, new MigrationJobRequest<>(handler, item.getEntityId())); + item.setMigrationScheduled(true); + } + public ExtensionVersion getExtension(long entityId) { return entityManager.find(ExtensionVersion.class, entityId); } From 914137964e9267375ce621024e74ba1bdf560402 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Mon, 6 Mar 2023 08:46:14 +0000 Subject: [PATCH 22/45] Add namespace route to frontedRoutes --- .../main/java/org/eclipse/openvsx/security/SecurityConfig.java | 2 +- server/src/main/java/org/eclipse/openvsx/web/WebConfig.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java b/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java index c2fe87faf..7fbf5214d 100644 --- a/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java +++ b/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java @@ -28,7 +28,7 @@ public class SecurityConfig extends WebSecurityConfigurerAdapter { @Autowired OAuth2UserServices userServices; - @Value("${ovsx.webui.frontendRoutes:/extension/**,/user-settings/**,/admin-dashboard/**}") + @Value("${ovsx.webui.frontendRoutes:/extension/**,/namespace/**,/user-settings/**,/admin-dashboard/**}") String[] frontendRoutes; @Override diff --git a/server/src/main/java/org/eclipse/openvsx/web/WebConfig.java b/server/src/main/java/org/eclipse/openvsx/web/WebConfig.java index 1a1873913..8835c85c6 100644 --- a/server/src/main/java/org/eclipse/openvsx/web/WebConfig.java +++ b/server/src/main/java/org/eclipse/openvsx/web/WebConfig.java @@ -30,7 +30,7 @@ public class WebConfig implements WebMvcConfigurer { @Value("${ovsx.webui.url:}") String webuiUrl; - @Value("${ovsx.webui.frontendRoutes:/extension/**,/user-settings/**,/admin-dashboard/**}") + @Value("${ovsx.webui.frontendRoutes:/extension/**,/namespace/**,/user-settings/**,/admin-dashboard/**}") String[] frontendRoutes; @Override From 5680c22e371e18a11357ab91301f23678f2210e8 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Thu, 9 Mar 2023 07:23:01 +0000 Subject: [PATCH 23/45] Create new StringBuilder --- .../org/eclipse/openvsx/web/LongRunningRequestFilter.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java b/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java index 0e4d95f05..477e5cc8a 100644 --- a/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java +++ b/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java @@ -26,11 +26,9 @@ public class LongRunningRequestFilter extends OncePerRequestFilter { private final long threshold; - private final StringBuilder builder; public LongRunningRequestFilter(long threshold) { this.threshold = threshold; - this.builder = new StringBuilder(); } @Override @@ -59,6 +57,7 @@ protected void doFilterInternal(HttpServletRequest request, HttpServletResponse } private void logWarning(HttpServletRequest request, HttpServletResponse response, long millis, long maxBytes, boolean hasJsonBody, boolean jsonBodyTooLong) throws IOException { + var builder = new StringBuilder(); builder.append("\n\t") .append(request.getMethod()) .append(" | ") @@ -98,6 +97,5 @@ private void logWarning(HttpServletRequest request, HttpServletResponse response .append(response.getStatus()); logger.warn(builder.toString()); - builder.setLength(0); } } From 03a5d038405d20f1436180daf00f8a4de5ea1221 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Thu, 9 Mar 2023 16:38:55 +0200 Subject: [PATCH 24/45] Log query string too --- .../eclipse/openvsx/web/LongRunningRequestFilter.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java b/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java index 477e5cc8a..a73f00460 100644 --- a/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java +++ b/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java @@ -61,8 +61,13 @@ private void logWarning(HttpServletRequest request, HttpServletResponse response builder.append("\n\t") .append(request.getMethod()) .append(" | ") - .append(request.getRequestURI()) - .append(" took ") + .append(request.getRequestURI()); + + if(request.getQueryString() != null) { + builder.append('?').append(request.getQueryString()); + } + + builder.append(" took ") .append(millis) .append(" ms.\n\t"); From 326f2e36c61873b612cbee9142f46251a5bff55b Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Wed, 1 Mar 2023 17:33:07 +0200 Subject: [PATCH 25/45] Corrupted metadata extension in some extensions regarding targetPlatform Added FixTargetPlatformMigration Remove extensionFile at end of a migration Don't run migrations in mirror mode --- .../ExtractResourcesJobRequestHandler.java | 3 + ...ExtractVsixManifestsJobRequestHandler.java | 5 ++ .../FixTargetPlatformsJobRequestHandler.java | 78 +++++++++++++++++++ .../migration/FixTargetPlatformsService.java | 40 ++++++++++ .../openvsx/migration/MigrationRunner.java | 7 ++ .../RenameDownloadsJobRequestHandler.java | 2 + .../SetPreReleaseJobRequestHandler.java | 5 ++ .../repositories/RepositoryService.java | 4 + ...4__ExtensionVersion_Fix_TargetPlatform.sql | 7 ++ .../RepositoryServiceSmokeTest.java | 1 + 10 files changed, 152 insertions(+) create mode 100644 server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java create mode 100644 server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsService.java create mode 100644 server/src/main/resources/db/migration/V1_34__ExtensionVersion_Fix_TargetPlatform.sql diff --git a/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java index 311ee8f8d..633e5a92b 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java @@ -19,6 +19,8 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import java.nio.file.Files; + @Component @ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "false", matchIfMissing = true) public class ExtractResourcesJobRequestHandler implements JobRequestHandler { @@ -50,5 +52,6 @@ public void run(MigrationJobRequest jobRequest) throws Exception { } service.deleteWebResources(extVersion); + Files.delete(extensionFile); } } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/ExtractVsixManifestsJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/ExtractVsixManifestsJobRequestHandler.java index 85130b8d4..848d38eaf 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/ExtractVsixManifestsJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/ExtractVsixManifestsJobRequestHandler.java @@ -17,11 +17,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import java.nio.file.Files; import java.util.AbstractMap; @Component +@ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "false", matchIfMissing = true) public class ExtractVsixManifestsJobRequestHandler implements JobRequestHandler { protected final Logger logger = new JobRunrDashboardLogger(LoggerFactory.getLogger(ExtractVsixManifestsJobRequestHandler.class)); @@ -50,5 +53,7 @@ public void run(MigrationJobRequest jobRequest) throws Exception { migrations.uploadFileResource(vsixManifest); migrations.persistFileResource(vsixManifest); } + + Files.delete(extensionFile); } } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java new file mode 100644 index 000000000..dc1b39e41 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java @@ -0,0 +1,78 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.migration; + +import org.eclipse.openvsx.AdminService; +import org.eclipse.openvsx.ExtensionProcessor; +import org.eclipse.openvsx.ExtensionService; +import org.eclipse.openvsx.entities.ExtensionVersion; +import org.jobrunr.jobs.annotations.Job; +import org.jobrunr.jobs.context.JobRunrDashboardLogger; +import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.nio.file.Files; +import java.util.AbstractMap; + +@Component +@ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "false", matchIfMissing = true) +public class FixTargetPlatformsJobRequestHandler implements JobRequestHandler { + + protected final Logger logger = new JobRunrDashboardLogger(LoggerFactory.getLogger(FixTargetPlatformsJobRequestHandler.class)); + + @Autowired + ExtensionService extensions; + + @Autowired + AdminService admins; + + @Autowired + MigrationService migrations; + + @Autowired + FixTargetPlatformsService service; + + @Override + @Job(name = "Fix target platform for published extension version", retries = 3) + public void run(MigrationJobRequest jobRequest) throws Exception { + var download = migrations.getResource(jobRequest); + var extVersion = download.getExtension(); + var content = migrations.getContent(download); + var extensionFile = migrations.getExtensionFile(new AbstractMap.SimpleEntry<>(download, content)); + try(var extProcessor = new ExtensionProcessor(extensionFile)) { + if(extProcessor.getMetadata().getTargetPlatform().equals(extVersion.getTargetPlatform())) { + return; + } + } + + logger.info("Fixing target platform for: {}.{}-{}@{}", extVersion.getExtension().getNamespace().getName(), extVersion.getExtension().getName(), extVersion.getVersion(), extVersion.getTargetPlatform()); + deleteExtension(extVersion); + try (var input = Files.newInputStream(extensionFile)) { + extensions.publishVersion(input, extVersion.getPublishedWith()); + } + + Files.delete(extensionFile); + } + + private void deleteExtension(ExtensionVersion extVersion) { + var extension = extVersion.getExtension(); + admins.deleteExtension( + extension.getNamespace().getName(), + extension.getName(), + extVersion.getTargetPlatform(), + extVersion.getVersion(), + service.getUser() + ); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsService.java b/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsService.java new file mode 100644 index 000000000..e8fe00547 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsService.java @@ -0,0 +1,40 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.migration; + +import org.eclipse.openvsx.entities.UserData; +import org.eclipse.openvsx.repositories.RepositoryService; +import org.springframework.stereotype.Component; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.persistence.EntityManager; +import javax.transaction.Transactional; + +@Component +public class FixTargetPlatformsService { + + @Autowired + RepositoryService repositories; + + @Autowired + EntityManager entityManager; + + @Transactional + public UserData getUser() { + var userName = "FixTargetPlatformMigration"; + var user = repositories.findUserByLoginName(null, userName); + if(user == null) { + user = new UserData(); + user.setLoginName(userName); + entityManager.persist(user); + } + return user; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java b/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java index fc898dfbe..6c7997cb3 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java @@ -35,6 +35,7 @@ public void run(HandlerJobRequest jobRequest) throws Exception { setPreReleaseMigration(); renameDownloadsMigration(); extractVsixManifestMigration(); + fixTargetPlatformMigration(); } private void extractResourcesMigration() { @@ -60,4 +61,10 @@ private void extractVsixManifestMigration() { var handler = ExtractVsixManifestsJobRequestHandler.class; repositories.findNotMigratedVsixManifests().forEach(item -> migrations.enqueueMigration(jobName, handler, item)); } + + private void fixTargetPlatformMigration() { + var jobName = "FixTargetPlatformMigration"; + var handler = FixTargetPlatformsJobRequestHandler.class; + repositories.findNotMigratedTargetPlatforms().forEach(item -> migrations.enqueueMigration(jobName, handler, item)); + } } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/RenameDownloadsJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/RenameDownloadsJobRequestHandler.java index 77fca12d0..7d8cb7995 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/RenameDownloadsJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/RenameDownloadsJobRequestHandler.java @@ -16,6 +16,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import java.nio.file.Files; import java.util.AbstractMap; @Component @@ -48,6 +49,7 @@ public void run(MigrationJobRequest jobRequest) throws Exception { download.setName(name); service.updateResource(download); + Files.delete(extensionFile); logger.info("Updated download name to: {}", name); } } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java index ac1227808..08d17a58b 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java @@ -15,9 +15,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import java.nio.file.Files; + @Component +@ConditionalOnProperty(value = "ovsx.data.mirror.enabled", havingValue = "false", matchIfMissing = true) public class SetPreReleaseJobRequestHandler implements JobRequestHandler { protected final Logger logger = new JobRunrDashboardLogger(LoggerFactory.getLogger(ExtractResourcesJobRequestHandler.class)); @@ -33,6 +37,7 @@ public void run(MigrationJobRequest jobRequest) throws Exception { var entry = service.getDownload(extVersion); var extensionFile = service.getExtensionFile(entry); service.updatePreviewAndPreRelease(extVersion, extensionFile); + Files.delete(extensionFile); } } } diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java index 166967ad1..eb52ec304 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java @@ -406,6 +406,10 @@ public Streamable findNotMigratedVsixManifests() { return findNotMigratedItems("V1_32__FileResource_Extract_VsixManifest.sql"); } + public Streamable findNotMigratedTargetPlatforms() { + return findNotMigratedItems("V1_34__ExtensionVersion_Fix_TargetPlatform.sql"); + } + private Streamable findNotMigratedItems(String migrationScript) { return migrationItemRepo.findByMigrationScriptAndMigrationScheduledFalseOrderById(migrationScript); } diff --git a/server/src/main/resources/db/migration/V1_34__ExtensionVersion_Fix_TargetPlatform.sql b/server/src/main/resources/db/migration/V1_34__ExtensionVersion_Fix_TargetPlatform.sql new file mode 100644 index 000000000..496891385 --- /dev/null +++ b/server/src/main/resources/db/migration/V1_34__ExtensionVersion_Fix_TargetPlatform.sql @@ -0,0 +1,7 @@ +INSERT INTO migration_item(id, migration_script, entity_id, migration_scheduled) +SELECT nextval('hibernate_sequence'), 'V1_34__ExtensionVersion_Fix_TargetPlatform.sql', fr.id, FALSE +FROM file_resource fr +JOIN extension_version ev ON ev.id = fr.extension_id +JOIN extension e ON e.id = ev.extension_id +WHERE fr.type = 'download' AND ev.target_platform = 'universal' +ORDER BY e.download_count DESC; \ No newline at end of file diff --git a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java index 52cdfffd1..a4394d1d4 100644 --- a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java +++ b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java @@ -147,6 +147,7 @@ void testExecuteQueries() { () -> repositories.findNotMigratedPreReleases(), () -> repositories.findNotMigratedRenamedDownloads(), () -> repositories.findNotMigratedVsixManifests(), + () -> repositories.findNotMigratedTargetPlatforms(), () -> repositories.topMostActivePublishingUsers(NOW, 1), () -> repositories.topNamespaceExtensions(NOW, 1), () -> repositories.topNamespaceExtensionVersions(NOW, 1), From 55810adea85c41ec5a268eb3f2b31e88d0f45a66 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Thu, 9 Mar 2023 18:24:42 +0200 Subject: [PATCH 26/45] Set logoBytes before storing it Improve social links UX --- .../eclipse/openvsx/ExtensionValidator.java | 4 +- .../java/org/eclipse/openvsx/UserService.java | 4 +- .../src/pages/user/user-namespace-details.tsx | 37 ++++++++++++++++--- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionValidator.java b/server/src/main/java/org/eclipse/openvsx/ExtensionValidator.java index 1a217161b..16b2f338a 100644 --- a/server/src/main/java/org/eclipse/openvsx/ExtensionValidator.java +++ b/server/src/main/java/org/eclipse/openvsx/ExtensionValidator.java @@ -70,7 +70,7 @@ public List validateNamespaceDetails(NamespaceDetailsJson json) { checkURL(json.supportLink, "supportLink", issues); var githubLink = json.socialLinks.get("github"); - if(githubLink != null && !githubLink.matches("https:\\/\\/www\\.github\\.com\\/[^\\/]+")) { + if(githubLink != null && !githubLink.matches("https:\\/\\/github\\.com\\/[^\\/]+")) { issues.add(new Issue("Invalid GitHub URL")); } var linkedinLink = json.socialLinks.get("linkedin"); @@ -78,7 +78,7 @@ public List validateNamespaceDetails(NamespaceDetailsJson json) { issues.add(new Issue("Invalid LinkedIn URL")); } var twitterLink = json.socialLinks.get("twitter"); - if(twitterLink != null && !twitterLink.matches("https:\\/\\/www\\.twitter\\.com\\/[^\\/]+")) { + if(twitterLink != null && !twitterLink.matches("https:\\/\\/twitter\\.com\\/[^\\/]+")) { issues.add(new Issue("Invalid Twitter URL")); } diff --git a/server/src/main/java/org/eclipse/openvsx/UserService.java b/server/src/main/java/org/eclipse/openvsx/UserService.java index 7f0177038..77c281992 100644 --- a/server/src/main/java/org/eclipse/openvsx/UserService.java +++ b/server/src/main/java/org/eclipse/openvsx/UserService.java @@ -241,14 +241,14 @@ public ResultJson updateNamespaceDetails(NamespaceDetailsJson details) { } namespace.setLogoName(details.logo); + namespace.setLogoBytes(details.logoBytes); storeNamespaceLogo(namespace); } else if (namespace.getLogoBytes() != null) { storageUtil.removeNamespaceLogo(namespace); namespace.setLogoName(null); + namespace.setLogoBytes(null); namespace.setLogoStorageType(null); } - - namespace.setLogoBytes(details.logoBytes); } return ResultJson.success("Updated details for namespace " + details.name); diff --git a/webui/src/pages/user/user-namespace-details.tsx b/webui/src/pages/user/user-namespace-details.tsx index 63c9bed7e..b3b358494 100644 --- a/webui/src/pages/user/user-namespace-details.tsx +++ b/webui/src/pages/user/user-namespace-details.tsx @@ -216,11 +216,11 @@ class UserNamespaceDetailsComponent extends React.Component Date: Thu, 9 Mar 2023 22:45:50 +0200 Subject: [PATCH 27/45] Bump webui to 0.9.3 --- webui/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/package.json b/webui/package.json index 95d15c939..2a1b83cb7 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "openvsx-webui", - "version": "0.9.2", + "version": "0.9.3", "description": "User interface for Eclipse Open VSX", "keywords": [ "react", From 8794eb8cf181e7d3aea433d9f9447b7b6c37bfa0 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Fri, 10 Mar 2023 16:06:09 +0200 Subject: [PATCH 28/45] Get icon from storage Use timestamp in namespace logo to bust browser cache --- .../eclipse/openvsx/ExtensionValidator.java | 3 +- .../java/org/eclipse/openvsx/UserService.java | 47 ++++++++++++++----- .../storage/AzureBlobStorageService.java | 12 +++++ .../storage/GoogleCloudStorageService.java | 21 +++++++++ .../openvsx/storage/IStorageService.java | 2 + .../openvsx/storage/StorageUtilService.java | 33 +++++++++++++ 6 files changed, 105 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionValidator.java b/server/src/main/java/org/eclipse/openvsx/ExtensionValidator.java index 16b2f338a..4d707824a 100644 --- a/server/src/main/java/org/eclipse/openvsx/ExtensionValidator.java +++ b/server/src/main/java/org/eclipse/openvsx/ExtensionValidator.java @@ -27,6 +27,7 @@ import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.json.NamespaceDetailsJson; import org.eclipse.openvsx.util.TargetPlatform; +import org.eclipse.openvsx.util.TimeUtil; import org.eclipse.openvsx.util.VersionAlias; import org.springframework.stereotype.Component; @@ -88,7 +89,7 @@ public List validateNamespaceDetails(NamespaceDetailsJson json) { var detectedType = tika.detect(in, json.logo); var logoType = MimeTypes.getDefaultMimeTypes().getRegisteredMimeType(detectedType); if(logoType != null) { - json.logo = "logo-" + json.name + logoType.getExtension(); + json.logo = "logo-" + json.name + "-" + System.currentTimeMillis() + logoType.getExtension(); if(!logoType.getType().equals(MediaType.image("png")) && !logoType.getType().equals(MediaType.image("jpg"))) { issues.add(new Issue("Namespace logo should be of png or jpg type")); } diff --git a/server/src/main/java/org/eclipse/openvsx/UserService.java b/server/src/main/java/org/eclipse/openvsx/UserService.java index 77c281992..7c3a0136a 100644 --- a/server/src/main/java/org/eclipse/openvsx/UserService.java +++ b/server/src/main/java/org/eclipse/openvsx/UserService.java @@ -9,18 +9,12 @@ ********************************************************************************/ package org.eclipse.openvsx; -import java.util.Arrays; -import java.util.Objects; -import java.util.UUID; - -import javax.persistence.EntityManager; -import javax.transaction.Transactional; - import com.google.common.base.Joiner; +import org.apache.commons.io.IOUtils; import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.entities.*; -import org.eclipse.openvsx.json.NamespaceDetailsJson; import org.eclipse.openvsx.json.AccessTokenJson; +import org.eclipse.openvsx.json.NamespaceDetailsJson; import org.eclipse.openvsx.json.ResultJson; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.security.IdPrincipal; @@ -35,6 +29,14 @@ import org.springframework.security.oauth2.core.user.OAuth2User; import org.springframework.stereotype.Component; +import javax.persistence.EntityManager; +import javax.transaction.Transactional; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Objects; +import java.util.UUID; + import static org.eclipse.openvsx.cache.CacheService.CACHE_NAMESPACE_DETAILS_JSON; import static org.eclipse.openvsx.util.UrlUtil.createApiUrl; @@ -234,16 +236,31 @@ public ResultJson updateNamespaceDetails(NamespaceDetailsJson details) { if(!Objects.equals(details.socialLinks, namespace.getSocialLinks())) { namespace.setSocialLinks(details.socialLinks); } - if(!Arrays.equals(details.logoBytes, namespace.getLogoBytes())) { - if (details.logoBytes != null) { - if (namespace.getLogoBytes() != null) { + if(details.logoBytes == null) { + details.logoBytes = new byte[0]; + } + + boolean contentEquals; + var oldLogo = storageUtil.downloadNamespaceLogo(namespace); + try ( + var newLogoInput = new ByteArrayInputStream(details.logoBytes); + var oldLogoInput = Files.newInputStream(oldLogo) + ) { + contentEquals = IOUtils.contentEquals(newLogoInput, oldLogoInput); + } catch (IOException e) { + throw new RuntimeException(e); + } + + if(!contentEquals) { + if (details.logoBytes.length > 0) { + if (namespace.getLogoStorageType() != null) { storageUtil.removeNamespaceLogo(namespace); } namespace.setLogoName(details.logo); namespace.setLogoBytes(details.logoBytes); storeNamespaceLogo(namespace); - } else if (namespace.getLogoBytes() != null) { + } else if (namespace.getLogoStorageType() != null) { storageUtil.removeNamespaceLogo(namespace); namespace.setLogoName(null); namespace.setLogoBytes(null); @@ -251,6 +268,12 @@ public ResultJson updateNamespaceDetails(NamespaceDetailsJson details) { } } + try { + Files.delete(oldLogo); + } catch (IOException e) { + throw new RuntimeException(e); + } + return ResultJson.success("Updated details for namespace " + details.name); } diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java index dca0cd965..b66ccbbce 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java @@ -26,6 +26,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.URI; +import java.nio.file.Files; import java.nio.file.Path; @Component @@ -189,4 +190,15 @@ public URI getNamespaceLogoLocation(Namespace namespace) { protected String getBlobName(Namespace namespace) { return UrlUtil.createApiUrl("", namespace.getName(), "logo", namespace.getLogoName()).substring(1); // remove first '/' } + + @Override + public Path downloadNamespaceLogo(Namespace namespace) { + try { + var logoFile = Files.createTempFile("namespace-logo", ".png"); + getContainerClient().getBlobClient(getBlobName(namespace)).downloadToFile(logoFile.toString(), true); + return logoFile; + } catch (IOException e) { + throw new RuntimeException(e); + } + } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java index d5fc46856..7f78b22a0 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; @@ -185,4 +186,24 @@ protected String getObjectId(Namespace namespace) { return UrlUtil.createApiUrl("", namespace.getName(), "logo", namespace.getLogoName()).substring(1); // remove first '/' } + @Override + public Path downloadNamespaceLogo(Namespace namespace) { + Path logoFile; + try { + logoFile = Files.createTempFile("namespace-logo", ".png"); + } catch (IOException e) { + throw new RuntimeException(e); + } + + try ( + var reader = getStorage().reader(BlobId.of(bucketId, getObjectId(namespace))); + var output = new FileOutputStream(logoFile.toFile()); + ) { + output.getChannel().transferFrom(reader, 0, Long.MAX_VALUE); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return logoFile; + } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java index 469671b18..991da607f 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java @@ -56,4 +56,6 @@ public interface IStorageService { * Returns the public access location of a namespace logo. */ URI getNamespaceLogoLocation(Namespace namespace); + + Path downloadNamespaceLogo(Namespace namespace); } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java index c55faa326..7988afba9 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java @@ -30,7 +30,9 @@ import javax.persistence.EntityManager; import javax.transaction.Transactional; +import java.io.IOException; import java.net.URI; +import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.concurrent.TimeUnit; @@ -221,6 +223,37 @@ public URI getNamespaceLogoLocation(Namespace namespace) { } } + public Path downloadNamespaceLogo(Namespace namespace) { + if(namespace.getLogoStorageType() == null) { + return createNamespaceLogoFile(); + } + + switch (namespace.getLogoStorageType()) { + case STORAGE_GOOGLE: + return googleStorage.downloadNamespaceLogo(namespace); + case STORAGE_AZURE: + return azureStorage.downloadNamespaceLogo(namespace); + case STORAGE_DB: + try { + var logoFile = createNamespaceLogoFile(); + Files.write(logoFile, namespace.getLogoBytes()); + return logoFile; + } catch (IOException e) { + throw new RuntimeException(e); + } + default: + return createNamespaceLogoFile(); + } + } + + private Path createNamespaceLogoFile() { + try { + return Files.createTempFile("namespace-logo", ".png"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private String getFileUrl(String name, ExtensionVersion extVersion, String serverUrl) { return UrlUtil.createApiFileUrl(serverUrl, extVersion, name); } From af019866b12eb85a6d6122e0c0988e45a4afed5a Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Wed, 15 Mar 2023 17:28:03 +0200 Subject: [PATCH 29/45] Delete temp file after use --- .../FixTargetPlatformsJobRequestHandler.java | 16 +++++++++------- .../storage/AzureDownloadCountService.java | 6 ++++++ .../eclipse/openvsx/ExtensionProcessorTest.java | 8 ++++++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java index dc1b39e41..316ba952f 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/FixTargetPlatformsJobRequestHandler.java @@ -50,16 +50,18 @@ public void run(MigrationJobRequest jobRequest) throws Exception { var extVersion = download.getExtension(); var content = migrations.getContent(download); var extensionFile = migrations.getExtensionFile(new AbstractMap.SimpleEntry<>(download, content)); + + boolean fixTargetPlatform; try(var extProcessor = new ExtensionProcessor(extensionFile)) { - if(extProcessor.getMetadata().getTargetPlatform().equals(extVersion.getTargetPlatform())) { - return; - } + fixTargetPlatform = !extProcessor.getMetadata().getTargetPlatform().equals(extVersion.getTargetPlatform()); } - logger.info("Fixing target platform for: {}.{}-{}@{}", extVersion.getExtension().getNamespace().getName(), extVersion.getExtension().getName(), extVersion.getVersion(), extVersion.getTargetPlatform()); - deleteExtension(extVersion); - try (var input = Files.newInputStream(extensionFile)) { - extensions.publishVersion(input, extVersion.getPublishedWith()); + if(fixTargetPlatform) { + logger.info("Fixing target platform for: {}.{}-{}@{}", extVersion.getExtension().getNamespace().getName(), extVersion.getExtension().getName(), extVersion.getVersion(), extVersion.getTargetPlatform()); + deleteExtension(extVersion); + try (var input = Files.newInputStream(extensionFile)) { + extensions.publishVersion(input, extVersion.getPublishedWith()); + } } Files.delete(extensionFile); diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java index 918358408..f4e03277d 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountService.java @@ -187,6 +187,12 @@ private Map> processBlobItem(String blobName) { .collect(Collectors.groupingBy(Map.Entry::getKey, Collectors.mapping(Map.Entry::getValue, Collectors.toList()))); } catch (IOException e) { throw new RuntimeException(e); + } finally { + try { + Files.delete(downloadsTempFile); + } catch (IOException e) { + logger.error("Failed to delete downloads file", e); + } } } diff --git a/server/src/test/java/org/eclipse/openvsx/ExtensionProcessorTest.java b/server/src/test/java/org/eclipse/openvsx/ExtensionProcessorTest.java index 4bef09b8d..ffaf7bd4e 100644 --- a/server/src/test/java/org/eclipse/openvsx/ExtensionProcessorTest.java +++ b/server/src/test/java/org/eclipse/openvsx/ExtensionProcessorTest.java @@ -42,6 +42,8 @@ void testTodoTree() throws Exception { checkResource(processor, FileResource.README, "README.md"); checkResource(processor, FileResource.ICON, "todo-tree.png"); checkResource(processor, FileResource.LICENSE, "License.txt"); + } finally { + Files.delete(path); } } @@ -50,6 +52,8 @@ void testChangelog() throws Exception { var path = writeToTempFile("util/changelog.zip"); try (var processor = new ExtensionProcessor(path)) { checkResource(processor, FileResource.CHANGELOG, "CHANGELOG.md"); + } finally { + Files.delete(path); } } @@ -60,6 +64,8 @@ void testCapitalizedCaseForResources() throws Exception { checkResource(processor, FileResource.CHANGELOG, "Changelog.md"); checkResource(processor, FileResource.README, "Readme.md"); checkResource(processor, FileResource.LICENSE, "License.txt"); + } finally { + Files.delete(path); } } @@ -70,6 +76,8 @@ void testMinorCaseForResources() throws Exception { checkResource(processor, FileResource.CHANGELOG, "changelog.md"); checkResource(processor, FileResource.README, "readme.md"); checkResource(processor, FileResource.LICENSE, "license.txt"); + } finally { + Files.delete(path); } } From 17b966c222a7f0dcd688a37262683c4a67f02841 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Wed, 15 Mar 2023 15:18:45 +0200 Subject: [PATCH 30/45] Deprecate `/api/-/query` POST endpoint --- .../eclipse/openvsx/IExtensionRegistry.java | 4 +- .../eclipse/openvsx/LocalRegistryService.java | 114 +++++++++--------- .../java/org/eclipse/openvsx/RegistryAPI.java | 61 +++------- .../openvsx/UpstreamRegistryService.java | 36 +++--- .../eclipse/openvsx/json/QueryParamJson.java | 22 +++- .../eclipse/openvsx/json/QueryRequest.java | 29 +++++ ...ryParamJsonV2.java => QueryRequestV2.java} | 17 +-- .../org/eclipse/openvsx/RegistryAPITest.java | 78 ++---------- 8 files changed, 159 insertions(+), 202 deletions(-) create mode 100644 server/src/main/java/org/eclipse/openvsx/json/QueryRequest.java rename server/src/main/java/org/eclipse/openvsx/json/{QueryParamJsonV2.java => QueryRequestV2.java} (56%) diff --git a/server/src/main/java/org/eclipse/openvsx/IExtensionRegistry.java b/server/src/main/java/org/eclipse/openvsx/IExtensionRegistry.java index f0b10308e..f01699600 100644 --- a/server/src/main/java/org/eclipse/openvsx/IExtensionRegistry.java +++ b/server/src/main/java/org/eclipse/openvsx/IExtensionRegistry.java @@ -30,9 +30,9 @@ public interface IExtensionRegistry { SearchResultJson search(ISearchService.Options options); - QueryResultJson query(QueryParamJson param); + QueryResultJson query(QueryRequest request); - QueryResultJson queryV2(QueryParamJsonV2 param); + QueryResultJson queryV2(QueryRequestV2 request); NamespaceDetailsJson getNamespaceDetails(String namespace); diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java index f92872d68..69cd46666 100644 --- a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java @@ -220,40 +220,40 @@ public SearchResultJson search(ISearchService.Options options) { } @Override - public QueryResultJson query(QueryParamJson param) { - if (!Strings.isNullOrEmpty(param.extensionId)) { - var split = param.extensionId.split("\\."); + public QueryResultJson query(QueryRequest request) { + if (!Strings.isNullOrEmpty(request.extensionId)) { + var split = request.extensionId.split("\\."); if (split.length != 2 || split[0].isEmpty() || split[1].isEmpty()) throw new ErrorResultException("The 'extensionId' parameter must have the format 'namespace.extension'."); - if (!Strings.isNullOrEmpty(param.namespaceName) && !param.namespaceName.equals(split[0])) + if (!Strings.isNullOrEmpty(request.namespaceName) && !request.namespaceName.equals(split[0])) throw new ErrorResultException("Conflicting parameters 'extensionId' and 'namespaceName'"); - if (!Strings.isNullOrEmpty(param.extensionName) && !param.extensionName.equals(split[1])) + if (!Strings.isNullOrEmpty(request.extensionName) && !request.extensionName.equals(split[1])) throw new ErrorResultException("Conflicting parameters 'extensionId' and 'extensionName'"); - param.namespaceName = split[0]; - param.extensionName = split[1]; + request.namespaceName = split[0]; + request.extensionName = split[1]; } List extensionVersions = new ArrayList<>(); - var targetPlatform = TargetPlatform.isValid(param.targetPlatform) ? param.targetPlatform : null; + var targetPlatform = TargetPlatform.isValid(request.targetPlatform) ? request.targetPlatform : null; // Add extension by UUID (public_id) - if (!Strings.isNullOrEmpty(param.extensionUuid)) { - extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionPublicId(targetPlatform, param.extensionUuid)); + if (!Strings.isNullOrEmpty(request.extensionUuid)) { + extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionPublicId(targetPlatform, request.extensionUuid)); } // Add extensions by namespace UUID (public_id) - if (!Strings.isNullOrEmpty(param.namespaceUuid)) { - extensionVersions.addAll(repositories.findActiveExtensionVersionsByNamespacePublicId(targetPlatform, param.namespaceUuid)); + if (!Strings.isNullOrEmpty(request.namespaceUuid)) { + extensionVersions.addAll(repositories.findActiveExtensionVersionsByNamespacePublicId(targetPlatform, request.namespaceUuid)); } // Add extension by namespace and name - if (!Strings.isNullOrEmpty(param.namespaceName) && !Strings.isNullOrEmpty(param.extensionName)) { - extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionName(targetPlatform, param.extensionName, param.namespaceName)); + if (!Strings.isNullOrEmpty(request.namespaceName) && !Strings.isNullOrEmpty(request.extensionName)) { + extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionName(targetPlatform, request.extensionName, request.namespaceName)); // Add extensions by namespace - } else if (!Strings.isNullOrEmpty(param.namespaceName)) { - extensionVersions.addAll(repositories.findActiveExtensionVersionsByNamespaceName(targetPlatform, param.namespaceName)); + } else if (!Strings.isNullOrEmpty(request.namespaceName)) { + extensionVersions.addAll(repositories.findActiveExtensionVersionsByNamespaceName(targetPlatform, request.namespaceName)); // Add extensions by name - } else if (!Strings.isNullOrEmpty(param.extensionName)) { - extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionName(targetPlatform, param.extensionName)); + } else if (!Strings.isNullOrEmpty(request.extensionName)) { + extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionName(targetPlatform, request.extensionName)); } extensionVersions = extensionVersions.stream() @@ -277,22 +277,22 @@ public QueryResultJson query(QueryParamJson param) { var membershipsByNamespaceId = getMemberships(extensionVersions); // Add a specific version of an extension - if (!Strings.isNullOrEmpty(param.namespaceName) && !Strings.isNullOrEmpty(param.extensionName) - && !Strings.isNullOrEmpty(param.extensionVersion) && !param.includeAllVersions) { + if (!Strings.isNullOrEmpty(request.namespaceName) && !Strings.isNullOrEmpty(request.extensionName) + && !Strings.isNullOrEmpty(request.extensionVersion) && !request.includeAllVersions) { extensionVersions = extensionVersions.stream() - .filter(ev -> ev.getVersion().equals(param.extensionVersion)) - .filter(ev -> ev.getExtension().getName().equals(param.extensionName)) - .filter(ev -> ev.getExtension().getNamespace().getName().equals(param.namespaceName)) + .filter(ev -> ev.getVersion().equals(request.extensionVersion)) + .filter(ev -> ev.getExtension().getName().equals(request.extensionName)) + .filter(ev -> ev.getExtension().getNamespace().getName().equals(request.namespaceName)) .collect(Collectors.toList()); } // Only add latest version of an extension - if(Strings.isNullOrEmpty(param.extensionVersion) && !param.includeAllVersions) { + if(Strings.isNullOrEmpty(request.extensionVersion) && !request.includeAllVersions) { extensionVersions = new ArrayList<>(latestVersions.values()); } var result = new QueryResultJson(); result.extensions = extensionVersions.stream() - .filter(ev -> addToResult(ev, param)) + .filter(ev -> addToResult(ev, request)) .sorted(getExtensionVersionComparator()) .map(ev -> { var latest = latestVersions.get(getLatestVersionKey(ev)); @@ -309,40 +309,40 @@ public QueryResultJson query(QueryParamJson param) { } @Override - public QueryResultJson queryV2(QueryParamJsonV2 param) { - if (!Strings.isNullOrEmpty(param.extensionId)) { - var split = param.extensionId.split("\\."); + public QueryResultJson queryV2(QueryRequestV2 request) { + if (!Strings.isNullOrEmpty(request.extensionId)) { + var split = request.extensionId.split("\\."); if (split.length != 2 || split[0].isEmpty() || split[1].isEmpty()) throw new ErrorResultException("The 'extensionId' parameter must have the format 'namespace.extension'."); - if (!Strings.isNullOrEmpty(param.namespaceName) && !param.namespaceName.equals(split[0])) + if (!Strings.isNullOrEmpty(request.namespaceName) && !request.namespaceName.equals(split[0])) throw new ErrorResultException("Conflicting parameters 'extensionId' and 'namespaceName'"); - if (!Strings.isNullOrEmpty(param.extensionName) && !param.extensionName.equals(split[1])) + if (!Strings.isNullOrEmpty(request.extensionName) && !request.extensionName.equals(split[1])) throw new ErrorResultException("Conflicting parameters 'extensionId' and 'extensionName'"); - param.namespaceName = split[0]; - param.extensionName = split[1]; + request.namespaceName = split[0]; + request.extensionName = split[1]; } List extensionVersions = new ArrayList<>(); - var targetPlatform = TargetPlatform.isValid(param.targetPlatform) ? param.targetPlatform : null; + var targetPlatform = TargetPlatform.isValid(request.targetPlatform) ? request.targetPlatform : null; // Add extension by UUID (public_id) - if (!Strings.isNullOrEmpty(param.extensionUuid)) { - extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionPublicId(targetPlatform, param.extensionUuid)); + if (!Strings.isNullOrEmpty(request.extensionUuid)) { + extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionPublicId(targetPlatform, request.extensionUuid)); } // Add extensions by namespace UUID (public_id) - if (!Strings.isNullOrEmpty(param.namespaceUuid)) { - extensionVersions.addAll(repositories.findActiveExtensionVersionsByNamespacePublicId(targetPlatform, param.namespaceUuid)); + if (!Strings.isNullOrEmpty(request.namespaceUuid)) { + extensionVersions.addAll(repositories.findActiveExtensionVersionsByNamespacePublicId(targetPlatform, request.namespaceUuid)); } // Add extension by namespace and name - if (!Strings.isNullOrEmpty(param.namespaceName) && !Strings.isNullOrEmpty(param.extensionName)) { - extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionName(targetPlatform, param.extensionName, param.namespaceName)); + if (!Strings.isNullOrEmpty(request.namespaceName) && !Strings.isNullOrEmpty(request.extensionName)) { + extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionName(targetPlatform, request.extensionName, request.namespaceName)); // Add extensions by namespace - } else if (!Strings.isNullOrEmpty(param.namespaceName)) { - extensionVersions.addAll(repositories.findActiveExtensionVersionsByNamespaceName(targetPlatform, param.namespaceName)); + } else if (!Strings.isNullOrEmpty(request.namespaceName)) { + extensionVersions.addAll(repositories.findActiveExtensionVersionsByNamespaceName(targetPlatform, request.namespaceName)); // Add extensions by name - } else if (!Strings.isNullOrEmpty(param.extensionName)) { - extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionName(targetPlatform, param.extensionName)); + } else if (!Strings.isNullOrEmpty(request.extensionName)) { + extensionVersions.addAll(repositories.findActiveExtensionVersionsByExtensionName(targetPlatform, request.extensionName)); } extensionVersions = extensionVersions.stream() @@ -367,27 +367,27 @@ public QueryResultJson queryV2(QueryParamJsonV2 param) { var membershipsByNamespaceId = getMemberships(extensionVersions); // Add a specific version of an extension - if (!Strings.isNullOrEmpty(param.namespaceName) && !Strings.isNullOrEmpty(param.extensionName) - && !Strings.isNullOrEmpty(param.extensionVersion) && !param.includeAllVersions.equals("true")) { + if (!Strings.isNullOrEmpty(request.namespaceName) && !Strings.isNullOrEmpty(request.extensionName) + && !Strings.isNullOrEmpty(request.extensionVersion) && !request.includeAllVersions.equals("true")) { extensionVersions = extensionVersions.stream() - .filter(ev -> ev.getVersion().equals(param.extensionVersion)) - .filter(ev -> ev.getExtension().getName().equals(param.extensionName)) - .filter(ev -> ev.getExtension().getNamespace().getName().equals(param.namespaceName)) + .filter(ev -> ev.getVersion().equals(request.extensionVersion)) + .filter(ev -> ev.getExtension().getName().equals(request.extensionName)) + .filter(ev -> ev.getExtension().getNamespace().getName().equals(request.namespaceName)) .collect(Collectors.toList()); } // Only add latest version of an extension - if(Strings.isNullOrEmpty(param.extensionVersion) && !param.includeAllVersions.equals("true")) { + if(Strings.isNullOrEmpty(request.extensionVersion) && !request.includeAllVersions.equals("true")) { extensionVersions = new ArrayList<>(latestVersions.values()); } // Revert to default includeAllVersions value when extensionVersion is set - if(!Strings.isNullOrEmpty(param.extensionVersion) && param.includeAllVersions.equals("true")) { - param.includeAllVersions = "links"; + if(!Strings.isNullOrEmpty(request.extensionVersion) && request.includeAllVersions.equals("true")) { + request.includeAllVersions = "links"; } - var addAllVersions = param.includeAllVersions.equals("links"); + var addAllVersions = request.includeAllVersions.equals("links"); var result = new QueryResultJson(); result.extensions = extensionVersions.stream() - .filter(ev -> addToResultV2(ev, param)) + .filter(ev -> addToResultV2(ev, request)) .sorted(getExtensionVersionComparator()) .map(ev -> { var latest = latestVersions.get(getLatestVersionKey(ev)); @@ -546,12 +546,12 @@ private Comparator getExtensionVersionComparator() { .thenComparing(ExtensionVersion.SORT_COMPARATOR); } - private boolean addToResult(ExtensionVersion extVersion, QueryParamJson param) { - return addToResult(extVersion, param.extensionVersion, param.extensionName, param.namespaceName, param.extensionUuid, param.namespaceUuid); + private boolean addToResult(ExtensionVersion extVersion, QueryRequest request) { + return addToResult(extVersion, request.extensionVersion, request.extensionName, request.namespaceName, request.extensionUuid, request.namespaceUuid); } - private boolean addToResultV2(ExtensionVersion extVersion, QueryParamJsonV2 param) { - return addToResult(extVersion, param.extensionVersion, param.extensionName, param.namespaceName, param.extensionUuid, param.namespaceUuid); + private boolean addToResultV2(ExtensionVersion extVersion, QueryRequestV2 request) { + return addToResult(extVersion, request.extensionVersion, request.extensionName, request.namespaceName, request.extensionUuid, request.namespaceUuid); } private boolean addToResult( diff --git a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java index 3bba8b244..8860ef76f 100644 --- a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java @@ -828,7 +828,7 @@ public ResponseEntity getQueryV2( return new ResponseEntity<>(json, HttpStatus.BAD_REQUEST); } - var param = new QueryParamJsonV2(); + var param = new QueryRequestV2(); param.namespaceName = namespaceName; param.extensionName = extensionName; param.extensionVersion = extensionVersion; @@ -932,20 +932,20 @@ public ResponseEntity getQuery( ) String targetPlatform ) { - var param = new QueryParamJson(); - param.namespaceName = namespaceName; - param.extensionName = extensionName; - param.extensionVersion = extensionVersion; - param.extensionId = extensionId; - param.extensionUuid = extensionUuid; - param.namespaceUuid = namespaceUuid; - param.includeAllVersions = includeAllVersions; - param.targetPlatform = targetPlatform; + var request = new QueryRequest(); + request.namespaceName = namespaceName; + request.extensionName = extensionName; + request.extensionVersion = extensionVersion; + request.extensionId = extensionId; + request.extensionUuid = extensionUuid; + request.namespaceUuid = namespaceUuid; + request.includeAllVersions = includeAllVersions; + request.targetPlatform = targetPlatform; var result = new QueryResultJson(); for (var registry : getRegistries()) { try { - var subResult = registry.query(param); + var subResult = registry.query(request); if (subResult.extensions != null) { if (result.extensions == null) result.extensions = subResult.extensions; @@ -969,19 +969,11 @@ public ResponseEntity getQuery( produces = MediaType.APPLICATION_JSON_VALUE ) @CrossOrigin - @Operation(summary = "Provides metadata of extensions matching the given parameters") + @Operation(summary = "Provides metadata of extensions matching the given parameters. Deprecated: use GET /api/-/query instead.", deprecated = true) @ApiResponses({ @ApiResponse( - responseCode = "200", - description = "Returns the (possibly empty) query results" - ), - @ApiResponse( - responseCode = "400", - description = "The request contains an invalid parameter value", - content = @Content( - mediaType = MediaType.APPLICATION_JSON_VALUE, - examples = @ExampleObject(value = "{\"error\":\"The 'extensionId' parameter must have the format 'namespace.extension'.\"}") - ) + responseCode = "301", + description = "Returns redirect to GET /api/-/query." ), @ApiResponse( responseCode = "429", @@ -1005,25 +997,12 @@ public ResponseEntity postQuery( @RequestBody @Parameter(description = "Parameters of the metadata query") QueryParamJson param ) { - var result = new QueryResultJson(); - for (var registry : getRegistries()) { - try { - var subResult = registry.query(param); - if (subResult.extensions != null) { - if (result.extensions == null) - result.extensions = subResult.extensions; - else - result.extensions.addAll(subResult.extensions); - } - } catch (NotFoundException exc) { - // Try the next registry - } catch (ErrorResultException exc) { - return exc.toResponseEntity(QueryResultJson.class); - } - } - return ResponseEntity.ok() - .cacheControl(CacheControl.maxAge(10, TimeUnit.MINUTES).cachePublic()) - .body(result); + var location = UrlUtil.createApiUrl(UrlUtil.getBaseUrl(), "api", "-", "query"); + location = UrlUtil.addQuery(location, param.toQueryParams()); + return ResponseEntity.status(HttpStatus.MOVED_PERMANENTLY) + .cacheControl(CacheControl.maxAge(1, TimeUnit.DAYS).cachePublic()) + .location(URI.create(location)) + .build(); } @PostMapping( diff --git a/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java b/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java index c3842dd97..44f6c52e1 100644 --- a/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/UpstreamRegistryService.java @@ -240,17 +240,17 @@ public SearchResultJson search(ISearchService.Options options) { } @Override - public QueryResultJson query(QueryParamJson param) { + public QueryResultJson query(QueryRequest request) { var urlTemplate = urlConfigService.getUpstreamUrl() + "/api/-/query"; var queryParams = new HashMap(); - queryParams.put("namespaceName", param.namespaceName); - queryParams.put("extensionName", param.extensionName); - queryParams.put("extensionVersion", param.extensionVersion); - queryParams.put("extensionId", param.extensionId); - queryParams.put("extensionUuid", param.extensionUuid); - queryParams.put("namespaceUuid", param.namespaceUuid); - queryParams.put("includeAllVersions", String.valueOf(param.includeAllVersions)); - queryParams.put("targetPlatform", param.targetPlatform); + queryParams.put("namespaceName", request.namespaceName); + queryParams.put("extensionName", request.extensionName); + queryParams.put("extensionVersion", request.extensionVersion); + queryParams.put("extensionId", request.extensionId); + queryParams.put("extensionUuid", request.extensionUuid); + queryParams.put("namespaceUuid", request.namespaceUuid); + queryParams.put("includeAllVersions", String.valueOf(request.includeAllVersions)); + queryParams.put("targetPlatform", request.targetPlatform); var queryString = queryParams.entrySet().stream() @@ -275,17 +275,17 @@ public QueryResultJson query(QueryParamJson param) { } @Override - public QueryResultJson queryV2(QueryParamJsonV2 param) { + public QueryResultJson queryV2(QueryRequestV2 request) { var urlTemplate = urlConfigService.getUpstreamUrl() + "/api/v2/-/query"; var queryParams = new HashMap(); - queryParams.put("namespaceName", param.namespaceName); - queryParams.put("extensionName", param.extensionName); - queryParams.put("extensionVersion", param.extensionVersion); - queryParams.put("extensionId", param.extensionId); - queryParams.put("extensionUuid", param.extensionUuid); - queryParams.put("namespaceUuid", param.namespaceUuid); - queryParams.put("includeAllVersions", String.valueOf(param.includeAllVersions)); - queryParams.put("targetPlatform", param.targetPlatform); + queryParams.put("namespaceName", request.namespaceName); + queryParams.put("extensionName", request.extensionName); + queryParams.put("extensionVersion", request.extensionVersion); + queryParams.put("extensionId", request.extensionId); + queryParams.put("extensionUuid", request.extensionUuid); + queryParams.put("namespaceUuid", request.namespaceUuid); + queryParams.put("includeAllVersions", String.valueOf(request.includeAllVersions)); + queryParams.put("targetPlatform", request.targetPlatform); var queryString = queryParams.entrySet().stream() .filter(entry -> !Strings.isNullOrEmpty(entry.getValue())) diff --git a/server/src/main/java/org/eclipse/openvsx/json/QueryParamJson.java b/server/src/main/java/org/eclipse/openvsx/json/QueryParamJson.java index fbf3d8977..7c6041052 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/QueryParamJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/QueryParamJson.java @@ -11,9 +11,12 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; - +import com.google.common.base.Strings; import io.swagger.v3.oas.annotations.media.Schema; +import java.util.LinkedHashMap; +import java.util.stream.Stream; + import static org.eclipse.openvsx.util.TargetPlatform.*; @Schema( @@ -52,4 +55,21 @@ public class QueryParamJson { NAME_WEB, NAME_UNIVERSAL }) public String targetPlatform; + + public String[] toQueryParams() { + var queryParams = new LinkedHashMap(); + queryParams.put("namespaceName", namespaceName); + queryParams.put("extensionName", extensionName); + queryParams.put("extensionVersion", extensionVersion); + queryParams.put("extensionId", extensionId); + queryParams.put("extensionUuid", extensionUuid); + queryParams.put("namespaceUuid", namespaceUuid); + queryParams.put("targetPlatform", targetPlatform); + queryParams.put("includeAllVersions", String.valueOf(includeAllVersions)); + + return queryParams.entrySet().stream() + .filter(entry -> !Strings.isNullOrEmpty(entry.getValue())) + .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue())) + .toArray(String[]::new); + } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/json/QueryRequest.java b/server/src/main/java/org/eclipse/openvsx/json/QueryRequest.java new file mode 100644 index 000000000..8399a833d --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/json/QueryRequest.java @@ -0,0 +1,29 @@ +/******************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ +package org.eclipse.openvsx.json; + +public class QueryRequest { + + public String namespaceName; + + public String extensionName; + + public String extensionVersion; + + public String extensionId; + + public String extensionUuid; + + public String namespaceUuid; + + public boolean includeAllVersions; + + public String targetPlatform; +} \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/json/QueryParamJsonV2.java b/server/src/main/java/org/eclipse/openvsx/json/QueryRequestV2.java similarity index 56% rename from server/src/main/java/org/eclipse/openvsx/json/QueryParamJsonV2.java rename to server/src/main/java/org/eclipse/openvsx/json/QueryRequestV2.java index 3cb7d387b..02b921ccd 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/QueryParamJsonV2.java +++ b/server/src/main/java/org/eclipse/openvsx/json/QueryRequestV2.java @@ -14,34 +14,21 @@ import io.swagger.v3.oas.annotations.media.Schema; -@Schema( - name = "QueryParamV2", - description = "Metadata query parameter version 2" -) -@JsonInclude(Include.NON_NULL) -public class QueryParamJsonV2 { - - @Schema(description = "Name of a namespace") +public class QueryRequestV2 { + public String namespaceName; - @Schema(description = "Name of an extension") public String extensionName; - @Schema(description = "Version of an extension") public String extensionVersion; - @Schema(description = "Identifier in the form {namespace}.{extension}") public String extensionId; - @Schema(description = "Universally unique identifier of an extension") public String extensionUuid; - @Schema(description = "Universally unique identifier of a namespace") public String namespaceUuid; - @Schema(description = "Whether to include all versions of an extension", allowableValues = { "true", "false", "links" }) public String includeAllVersions; - @Schema(description = "Name of the target platform") public String targetPlatform; } \ No newline at end of file diff --git a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java index 9628ca3c3..8fd02aef1 100644 --- a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java @@ -1030,15 +1030,8 @@ public void testPostQueryExtensionName() throws Exception { mockMvc.perform(post("/api/-/query") .contentType(MediaType.APPLICATION_JSON) .content("{ \"extensionName\": \"bar\" }")) - .andExpect(status().isOk()) - .andExpect(content().json(queryResultJson(e -> { - e.namespace = "foo"; - e.name = "bar"; - e.version = "1.0.0"; - e.verified = false; - e.timestamp = "2000-01-01T10:00Z"; - e.displayName = "Foo Bar"; - }))); + .andExpect(status().isMovedPermanently()) + .andExpect(header().string("Location", "http://localhost/api/-/query?extensionName=bar&includeAllVersions=false")); } @Test @@ -1047,38 +1040,8 @@ public void testPostQueryNamespace() throws Exception { mockMvc.perform(post("/api/-/query") .contentType(MediaType.APPLICATION_JSON) .content("{ \"namespaceName\": \"foo\" }")) - .andExpect(status().isOk()) - .andExpect(content().json(queryResultJson(e -> { - e.namespace = "foo"; - e.name = "bar"; - e.version = "1.0.0"; - e.verified = false; - e.timestamp = "2000-01-01T10:00Z"; - e.displayName = "Foo Bar"; - }))); - } - - @Test - public void testPostQueryUnknownExtension() throws Exception { - mockExtensionVersion(); - Mockito.when(repositories.findActiveExtensionVersionsByExtensionName(null, "baz")) - .thenReturn(Collections.emptyList()); - - mockMvc.perform(post("/api/-/query") - .contentType(MediaType.APPLICATION_JSON) - .content("{ \"extensionName\": \"baz\" }")) - .andExpect(status().isOk()) - .andExpect(content().json("{ \"extensions\": [] }")); - } - - @Test - public void testPostQueryInactiveExtension() throws Exception { - mockInactiveExtensionVersion("foo", "bar"); - mockMvc.perform(post("/api/-/query") - .contentType(MediaType.APPLICATION_JSON) - .content("{ \"extensionId\": \"foo.bar\" }")) - .andExpect(status().isOk()) - .andExpect(content().json("{ \"extensions\": [] }")); + .andExpect(status().isMovedPermanently()) + .andExpect(header().string("Location", "http://localhost/api/-/query?namespaceName=foo&includeAllVersions=false")); } @Test @@ -1087,15 +1050,8 @@ public void testPostQueryExtensionId() throws Exception { mockMvc.perform(post("/api/-/query") .contentType(MediaType.APPLICATION_JSON) .content("{ \"extensionId\": \"foo.bar\" }")) - .andExpect(status().isOk()) - .andExpect(content().json(queryResultJson(e -> { - e.namespace = "foo"; - e.name = "bar"; - e.version = "1.0.0"; - e.verified = false; - e.timestamp = "2000-01-01T10:00Z"; - e.displayName = "Foo Bar"; - }))); + .andExpect(status().isMovedPermanently()) + .andExpect(header().string("Location", "http://localhost/api/-/query?extensionId=foo.bar&includeAllVersions=false")); } @Test @@ -1104,15 +1060,8 @@ public void testPostQueryExtensionUuid() throws Exception { mockMvc.perform(post("/api/-/query") .contentType(MediaType.APPLICATION_JSON) .content("{ \"extensionUuid\": \"5678\" }")) - .andExpect(status().isOk()) - .andExpect(content().json(queryResultJson(e -> { - e.namespace = "foo"; - e.name = "bar"; - e.version = "1.0.0"; - e.verified = false; - e.timestamp = "2000-01-01T10:00Z"; - e.displayName = "Foo Bar"; - }))); + .andExpect(status().isMovedPermanently()) + .andExpect(header().string("Location", "http://localhost/api/-/query?extensionUuid=5678&includeAllVersions=false")); } @Test @@ -1121,15 +1070,8 @@ public void testPostQueryNamespaceUuid() throws Exception { mockMvc.perform(post("/api/-/query") .contentType(MediaType.APPLICATION_JSON) .content("{ \"namespaceUuid\": \"1234\" }")) - .andExpect(status().isOk()) - .andExpect(content().json(queryResultJson(e -> { - e.namespace = "foo"; - e.name = "bar"; - e.version = "1.0.0"; - e.verified = false; - e.timestamp = "2000-01-01T10:00Z"; - e.displayName = "Foo Bar"; - }))); + .andExpect(status().isMovedPermanently()) + .andExpect(header().string("Location", "http://localhost/api/-/query?namespaceUuid=1234&includeAllVersions=false")); } @Test From 8f1319c235e5cf589cb7193cd666480de0b7a806 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Thu, 16 Mar 2023 07:40:40 +0200 Subject: [PATCH 31/45] Move around some imports to rebuild postQuery Docker image --- .../java/org/eclipse/openvsx/RegistryAPI.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java index 8860ef76f..448470e7f 100644 --- a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java @@ -9,14 +9,7 @@ ********************************************************************************/ package org.eclipse.openvsx; -import java.io.InputStream; -import java.net.URI; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; - import com.google.common.collect.Iterables; - import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.headers.Header; @@ -30,17 +23,19 @@ import org.eclipse.openvsx.util.*; import org.elasticsearch.common.Strings; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.*; -import org.springframework.web.bind.annotation.CrossOrigin; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.http.CacheControl; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; import org.springframework.web.server.ResponseStatusException; import javax.servlet.http.HttpServletRequest; +import java.io.InputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; import static org.eclipse.openvsx.util.TargetPlatform.*; From 5458a8ea65d6a2484a0312fd248cddc4fc98004a Mon Sep 17 00:00:00 2001 From: David Xia Date: Fri, 17 Mar 2023 21:45:15 -0400 Subject: [PATCH 32/45] fix docs: only use straight quotes Currently copy-pasting some commands with smart quotes results in errors. Replace all the smart quotes with straight quotes for consistency. Signed-off-by: David Xia --- doc/development.md | 8 ++-- .../static/documents/publisher-agreement.md | 40 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/doc/development.md b/doc/development.md index 4aa02323d..249a6c179 100644 --- a/doc/development.md +++ b/doc/development.md @@ -14,12 +14,12 @@ To get started quickly, it is recommended to use Gitpod as default with the deve - Make sure the firewall is not blocking internet access on WSL2 - need to check every time before running the application or installing new tools. - - Try to ping a random website, for example, “ping -c 3 [www.google.ca](http://www.google.ca)”, if fails, proceed below + - Try to ping a random website, for example, "ping -c 3 [www.google.ca](http://www.google.ca)", if fails, proceed below - sudo vi /etc/resolv.conf - Change nameserver to 8.8.8.8 - Ping google again to see if it works this time - Related issue: [https://github.com/microsoft/WSL/issues/3669](https://github.com/microsoft/WSL/issues/3669) - - Solution that I’m following: [https://stackoverflow.com/questions/60269422/windows10-wsl2-ubuntu-debian-no-network](https://stackoverflow.com/questions/60269422/windows10-wsl2-ubuntu-debian-no-network) + - Solution that I'm following: [https://stackoverflow.com/questions/60269422/windows10-wsl2-ubuntu-debian-no-network](https://stackoverflow.com/questions/60269422/windows10-wsl2-ubuntu-debian-no-network) - clone the repository from github - set up vscode on WSL @@ -33,7 +33,7 @@ To get started quickly, it is recommended to use Gitpod as default with the deve - sudo service postgresql start - sudo service postgresql status (this is to check if the service is really on) - sudo -u postgres psql (connect to the postgres service) - - CREATE ROLE gitpod with LOGIN PASSWORD ‘gitpod’; + - CREATE ROLE gitpod with LOGIN PASSWORD 'gitpod'; - [https://learn.microsoft.com/en-us/windows/wsl/tutorials/wsl-database](https://learn.microsoft.com/en-us/windows/wsl/tutorials/wsl-database) - Install node.js using nvm @@ -52,7 +52,7 @@ To get started quickly, it is recommended to use Gitpod as default with the deve - Instal Elasticsearch with Docker - sudo docker pull elasticsearch:7.9.3 - - sudo docker run -d -name elasticsearch -p 9200:9200 -p 9300:9300 -e “discovery.type=single-node” elasticsearch:7.9.3 + - sudo docker run -d -name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.9.3 - cd cli diff --git a/server/src/dev/resources/static/documents/publisher-agreement.md b/server/src/dev/resources/static/documents/publisher-agreement.md index 3458e22ab..6383724cc 100644 --- a/server/src/dev/resources/static/documents/publisher-agreement.md +++ b/server/src/dev/resources/static/documents/publisher-agreement.md @@ -2,47 +2,47 @@ # Open VSX Publisher Agreement -Thank you for your interest in publishing in the Eclipse Open VSX Registry (“Open VSX”). As used in this Eclipse Open VSX Publisher Agreement (“Agreement”), the term “you”, ”your” or “Publisher” refers to you as an individual, and, to the extent you are the agent or employee of another person or entity that has rights in the Offering (as defined below) (such person or entity, a “Principal”) “you” and “publisher” also includes that Principal. +Thank you for your interest in publishing in the Eclipse Open VSX Registry ("Open VSX"). As used in this Eclipse Open VSX Publisher Agreement ("Agreement"), the term "you", "your" or "Publisher" refers to you as an individual, and, to the extent you are the agent or employee of another person or entity that has rights in the Offering (as defined below) (such person or entity, a "Principal") "you" and "publisher" also includes that Principal. -This Agreement describes the relationship between you and the Eclipse Foundation, Inc. (“Eclipse”, “us” or “we”) and governs your publication of any Offering (as defined below) within the Registry (as defined below). +This Agreement describes the relationship between you and the Eclipse Foundation, Inc. ("Eclipse", "us" or "we") and governs your publication of any Offering (as defined below) within the Registry (as defined below). By clicking where indicated to accept this Agreement, the individual entering into this Agreement represents and warrants to Eclipse that such individual has the authority to enter into this Agreement (including on behalf of his or her Principal), and you (including such individual and any Principal) agree to be bound by its terms. ## SECTION 1 Definitions. -a. **“Code Content”** shall have the meaning set forth in the [Eclipse Intellectual Property Policy](https://www.eclipse.org/org/documents/Eclipse_IP_Policy.pdf). +a. **"Code Content"** shall have the meaning set forth in the [Eclipse Intellectual Property Policy](https://www.eclipse.org/org/documents/Eclipse_IP_Policy.pdf). -b. **“Content”** shall have the meaning set forth in the [Eclipse Intellectual Property Policy](https://www.eclipse.org/org/documents/Eclipse_IP_Policy.pdf). +b. **"Content"** shall have the meaning set forth in the [Eclipse Intellectual Property Policy](https://www.eclipse.org/org/documents/Eclipse_IP_Policy.pdf). -c. **“Listing Information”** means: (i) the information and images accompanying an Offering that identifies the original source of the Offering, as well as the nature and other features of the Offering, as specified by you in connection with your request to publish the Offering; and (ii) to the extent applicable, Data Information (as defined herein). +c. **"Listing Information"** means: (i) the information and images accompanying an Offering that identifies the original source of the Offering, as well as the nature and other features of the Offering, as specified by you in connection with your request to publish the Offering; and (ii) to the extent applicable, Data Information (as defined herein). -d. **“Non-Code Content”** shall have the meaning set forth in the [Eclipse Intellectual Property Policy](https://www.eclipse.org/org/documents/Eclipse_IP_Policy.pdf). +d. **"Non-Code Content"** shall have the meaning set forth in the [Eclipse Intellectual Property Policy](https://www.eclipse.org/org/documents/Eclipse_IP_Policy.pdf). -e. **“Offering”** means any software, data, media, documentation, etc. published or proposed to be published in the Registry under this Agreement. +e. **"Offering"** means any software, data, media, documentation, etc. published or proposed to be published in the Registry under this Agreement. -f. **“Offering Contents”** means all data and software included within, installable by, or otherwise associated with an Offering. Offering Contents include, without limitation, all operating system and application software associated with an Offering. +f. **"Offering Contents"** means all data and software included within, installable by, or otherwise associated with an Offering. Offering Contents include, without limitation, all operating system and application software associated with an Offering. -g. **“Publisher Account”** means a publisher account for Open VSX, which includes a username and password. +g. **"Publisher Account"** means a publisher account for Open VSX, which includes a username and password. -h. **“Registry”** means a limited, Eclipse Hosted repository of Offerings published to [Open VSX](https://open-vsx.org/). +h. **"Registry"** means a limited, Eclipse Hosted repository of Offerings published to [Open VSX](https://open-vsx.org/). -i. **“Terms of Use”** means the [Eclipse.org Terms of Use](https://www.eclipse.org/legal/termsofuse.php), the legal terms under which you grant others the right to use or access your Offering, as well as all Offering Contents associated therewith, as specified in the Listing Information associated with your Offering. +i. **"Terms of Use"** means the [Eclipse.org Terms of Use](https://www.eclipse.org/legal/termsofuse.php), the legal terms under which you grant others the right to use or access your Offering, as well as all Offering Contents associated therewith, as specified in the Listing Information associated with your Offering. j. All other capitalized terms that are not defined in this Section 1 shall have the meanings assigned in the text of this Agreement. ## SECTION 2 Publisher Account. -Your Publisher Account is only for your use, and you are responsible for all activity that takes place within your Publisher Account. If you fail to keep your Publisher Account in good standing (for example by providing incorrect or outdated information, by engaging in dishonest or fraudulent activity, or by repeatedly submitting Offerings that violate this Agreement, abuse the Registry or interfere with any other party’s use of the Registry), we may revoke your Publisher Account, remove your Offerings from the Registry, delete Offering ratings and reviews, and pursue any other remedies available to us. +Your Publisher Account is only for your use, and you are responsible for all activity that takes place within your Publisher Account. If you fail to keep your Publisher Account in good standing (for example by providing incorrect or outdated information, by engaging in dishonest or fraudulent activity, or by repeatedly submitting Offerings that violate this Agreement, abuse the Registry or interfere with any other party's use of the Registry), we may revoke your Publisher Account, remove your Offerings from the Registry, delete Offering ratings and reviews, and pursue any other remedies available to us. ## SECTION 3 Submission, Approval, and Publication of Offerings. -a. **Submission Process.** You must submit a request for each Offering that you wish to publish in the Registry. We may approve or reject any proposed Offering in our sole discretion, and may condition our approval on your making modifications to the Offering or its Listing Information. You are responsible for ensuring that the Listing Information associated with your Offering is accurate and not misleading and does not violate third parties’ intellectual property rights, including third-party trademarks or icons. We may require you to provide us with one or more Offering prototypes. Following our approval of an Offering, you may publish the Offering in the Registry, subject to the terms and conditions of this Agreement and the Listing Information provided with your request. Publications of any Offering in the Registry is subject to this agreement being executed and being in effect at the time of publication. +a. **Submission Process.** You must submit a request for each Offering that you wish to publish in the Registry. We may approve or reject any proposed Offering in our sole discretion, and may condition our approval on your making modifications to the Offering or its Listing Information. You are responsible for ensuring that the Listing Information associated with your Offering is accurate and not misleading and does not violate third parties' intellectual property rights, including third-party trademarks or icons. We may require you to provide us with one or more Offering prototypes. Following our approval of an Offering, you may publish the Offering in the Registry, subject to the terms and conditions of this Agreement and the Listing Information provided with your request. Publications of any Offering in the Registry is subject to this agreement being executed and being in effect at the time of publication. b. **Presentation of Offerings.** We reserve the right to determine the manner in which all Offerings, whether published by you or others, are presented and promoted in the Registry. We may display your Listing Information in connection with your Offering, as well as other information designed to inform that the Offering is provided by you, including what content is included within the Offering. Notwithstanding our approval of an Offering as described in Section 3(a) above, we, in our sole discretion may determine not to make an Offering available in the Registry. -c. **Ratings and Reviews.** You understand that as part of the Registry, we may make available a facility for third parties to post ratings and reviews on your Offering (“Reviews”). You understand that while we may (but are not obligated to) exercise traditional editorial functions associated with such Reviews, we have no obligation to, and shall not be, reviewing, vetting, or otherwise examining such Reviews and have no obligation to remove Reviews that may be unfavorable to your Offering. +c. **Ratings and Reviews.** You understand that as part of the Registry, we may make available a facility for third parties to post ratings and reviews on your Offering ("Reviews"). You understand that while we may (but are not obligated to) exercise traditional editorial functions associated with such Reviews, we have no obligation to, and shall not be, reviewing, vetting, or otherwise examining such Reviews and have no obligation to remove Reviews that may be unfavorable to your Offering. -d. **Terms for Publisher Marks.** You hereby grant us a non-exclusive, royalty-free, personal license to display the trademarks and logos associated with the Offering (“Publisher Marks”), as provided to us in connection with the marketing and promotion of your Offerings in the Registry. You represent and warrant that you are the owner and/or authorized licensor of the Publisher Marks. As between the parties, all goodwill associated with the Publisher Marks shall inure to your benefit. We may reformat or resize Publisher Marks as necessary and without altering the overall appearance of the Publisher Marks. +d. **Terms for Publisher Marks.** You hereby grant us a non-exclusive, royalty-free, personal license to display the trademarks and logos associated with the Offering ("Publisher Marks"), as provided to us in connection with the marketing and promotion of your Offerings in the Registry. You represent and warrant that you are the owner and/or authorized licensor of the Publisher Marks. As between the parties, all goodwill associated with the Publisher Marks shall inure to your benefit. We may reformat or resize Publisher Marks as necessary and without altering the overall appearance of the Publisher Marks. e. **No Compensation.** You expressly acknowledge that neither Eclipse nor any licensee or end-user of any of your Offerings published on the Registry shall be required to provide you with any compensation for the distribution or use of your Offering as made available on the Registry. @@ -56,7 +56,7 @@ c. **Enforcement/Monitoring.** We are not responsible for monitoring the use of ## SECTION 5 Privacy and Data Protection. -To the extent that you collect any data regarding the use of your Offering, including without limitation any information regarding the licensee or end-user of the Offering, you shall disclose in your Listing Information for the Offering a full and complete description of what data you collect, for what purposes it is used, with whom it is shared and how long it is retained (“Data Information”). Without limitation of the foregoing, you agree to comply with all applicable data protection and privacy laws, regulations and ordinances relevant to the use of your Offering. +To the extent that you collect any data regarding the use of your Offering, including without limitation any information regarding the licensee or end-user of the Offering, you shall disclose in your Listing Information for the Offering a full and complete description of what data you collect, for what purposes it is used, with whom it is shared and how long it is retained ("Data Information"). Without limitation of the foregoing, you agree to comply with all applicable data protection and privacy laws, regulations and ordinances relevant to the use of your Offering. ## SECTION 6 Removal of Offerings. @@ -86,11 +86,11 @@ a. **DISCLAIMER OF WARRANTY.** WE PROVIDE THE REGISTRY "AS-IS," "WITH ALL FAULTS b. **Limitation of Liability.** TO THE EXTENT PERMITTED BY APPLICABLE LAW, ECLIPSE SHALL NOT HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE OFFERING OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. -c. **Duty to Defend.** You agree to defend, indemnify, and hold harmless us, Eclipse members, agents, officers, directors and employees as applicable, from and against any costs, losses, damages, liabilities or expenses and attorneys’ fees arising from any and all third-party claims alleging that your Offering, including any Listing Information, infringes any proprietary or personal right of a third party; or arising from any dispute between you and a licensee or End User of your Offering, relating to your Offering. +c. **Duty to Defend.** You agree to defend, indemnify, and hold harmless us, Eclipse members, agents, officers, directors and employees as applicable, from and against any costs, losses, damages, liabilities or expenses and attorneys' fees arising from any and all third-party claims alleging that your Offering, including any Listing Information, infringes any proprietary or personal right of a third party; or arising from any dispute between you and a licensee or End User of your Offering, relating to your Offering. ## SECTION 9 Term and Termination. -a. **General.** This Agreement will remain in effect until terminated. Either party may terminate this Agreement at any time, for any reason or no reason, upon thirty (30) days’ written notice and removal of all of your Offerings from the Registry. +a. **General.** This Agreement will remain in effect until terminated. Either party may terminate this Agreement at any time, for any reason or no reason, upon thirty (30) days' written notice and removal of all of your Offerings from the Registry. b. **Effect of Termination.** Sections of this Agreement that, by their terms, require performance or establish rights or protections after the termination or expiration of this Agreement will survive. @@ -104,11 +104,11 @@ c. **Jurisdiction and Governing Law.** This Agreement will be governed by the la d. **Responding to Claims.** If we receive a claim from a third party requesting that your Offering be changed or removed, we may refer that claim to you. If you believe that your Offering may be in violation of the terms of this Agreement, you must immediately notify us and work with us to cure the violation. -e. **Waiver.** Either party’s delay or failure to exercise any right or remedy will not result in a waiver of that or any other right or remedy +e. **Waiver.** Either party's delay or failure to exercise any right or remedy will not result in a waiver of that or any other right or remedy f. **Severability.** If any court of competent jurisdiction determines that any provision of this Agreement is illegal, invalid, or unenforceable, the remaining provisions will remain in full force and effect. -g. **Assignment.** Except as provided for below in this paragraph, neither party may assign this Agreement (or any rights or duties under it) without the other party’s prior written consent, provided that either party may assign this Agreement without the other party’s consent in connection with a merger, acquisition, or sale or transfer of all or substantially all of its assets, excepting that Eclipse may assign this agreement to any entity that serves as a successor steward to the Registry. Either party who assigns this Agreement as permitted in this Section 10(g) shall provide the other party with prompt notice of such assignment. Subject to the foregoing, this Agreement will be binding upon and inure to the benefit of the parties and their permitted successors and assigns. +g. **Assignment.** Except as provided for below in this paragraph, neither party may assign this Agreement (or any rights or duties under it) without the other party's prior written consent, provided that either party may assign this Agreement without the other party's consent in connection with a merger, acquisition, or sale or transfer of all or substantially all of its assets, excepting that Eclipse may assign this agreement to any entity that serves as a successor steward to the Registry. Either party who assigns this Agreement as permitted in this Section 10(g) shall provide the other party with prompt notice of such assignment. Subject to the foregoing, this Agreement will be binding upon and inure to the benefit of the parties and their permitted successors and assigns. h. **English Language.** The parties intend for this Agreement to be written and interpreted solely in English. Any notices required or provided under this Agreement will be in English. In the event of any conflict between the English version of this Agreement or any notices and a translation of the same, the English version will prevail. From 06234cec14c374cf37609577da55696a7f2fddd4 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Wed, 15 Mar 2023 12:26:51 +0200 Subject: [PATCH 33/45] Renaming a namespace appears to leave incorrect URLs. server: move files to new namespace in background job webui: show confirmation dialog, instead of loading new namespace --- .../java/org/eclipse/openvsx/AdminAPI.java | 2 +- .../org/eclipse/openvsx/AdminService.java | 67 ++---- .../openvsx/ChangeNamespaceJobRequest.java | 53 +++++ .../ChangeNamespaceJobRequestHandler.java | 195 ++++++++++++++++++ .../openvsx/ChangeNamespaceService.java | 86 ++++++++ .../eclipse/openvsx/ExtensionProcessor.java | 2 +- .../openvsx/json/ChangeNamespaceJson.java | 15 ++ .../repositories/FileResourceRepository.java | 3 + .../repositories/RepositoryService.java | 4 + .../storage/AzureBlobStorageService.java | 27 +++ .../storage/GoogleCloudStorageService.java | 16 ++ .../openvsx/storage/IStorageService.java | 4 + .../openvsx/storage/StorageUtilService.java | 13 ++ .../org/eclipse/openvsx/AdminAPITest.java | 127 ++---------- .../RepositoryServiceSmokeTest.java | 3 +- webui/src/components/info-dialog.tsx | 70 +++++++ .../pages/admin-dashboard/namespace-admin.tsx | 1 - .../namespace-change-dialog.tsx | 9 +- .../user/user-settings-namespace-detail.tsx | 6 +- 19 files changed, 532 insertions(+), 171 deletions(-) create mode 100644 server/src/main/java/org/eclipse/openvsx/ChangeNamespaceJobRequest.java create mode 100644 server/src/main/java/org/eclipse/openvsx/ChangeNamespaceJobRequestHandler.java create mode 100644 server/src/main/java/org/eclipse/openvsx/ChangeNamespaceService.java create mode 100644 webui/src/components/info-dialog.tsx diff --git a/server/src/main/java/org/eclipse/openvsx/AdminAPI.java b/server/src/main/java/org/eclipse/openvsx/AdminAPI.java index b1089ad0e..baf2f6071 100644 --- a/server/src/main/java/org/eclipse/openvsx/AdminAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/AdminAPI.java @@ -282,7 +282,7 @@ public ResponseEntity changeNamespace(@RequestBody ChangeNamespaceJs try { admins.checkAdminUser(); admins.changeNamespace(json); - return ResponseEntity.ok(ResultJson.success("Changed namespace " + json.oldNamespace + " to " + json.newNamespace)); + return ResponseEntity.ok(ResultJson.success("Scheduled namespace change from '" + json.oldNamespace + "' to '" + json.newNamespace + "'.\nIt can take 15 minutes to a couple hours for the change to become visible.")); } catch (ErrorResultException exc) { return exc.toResponseEntity(); } diff --git a/server/src/main/java/org/eclipse/openvsx/AdminService.java b/server/src/main/java/org/eclipse/openvsx/AdminService.java index db1910b9e..13337b584 100644 --- a/server/src/main/java/org/eclipse/openvsx/AdminService.java +++ b/server/src/main/java/org/eclipse/openvsx/AdminService.java @@ -9,17 +9,7 @@ ********************************************************************************/ package org.eclipse.openvsx; -import java.time.DateTimeException; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; -import java.util.*; -import java.util.stream.Collectors; - -import javax.persistence.EntityManager; -import javax.transaction.Transactional; - import com.google.common.base.Strings; - import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.eclipse.EclipseService; import org.eclipse.openvsx.entities.*; @@ -31,10 +21,20 @@ import org.eclipse.openvsx.util.TimeUtil; import org.eclipse.openvsx.util.UrlUtil; import org.eclipse.openvsx.util.VersionService; +import org.jobrunr.scheduling.JobRequestScheduler; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; +import javax.persistence.EntityManager; +import javax.transaction.Transactional; +import java.time.DateTimeException; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.stream.Collectors; + import static org.eclipse.openvsx.entities.FileResource.*; @Component @@ -70,6 +70,9 @@ public class AdminService { @Autowired CacheService cache; + @Autowired + JobRequestScheduler scheduler; + @Transactional(rollbackOn = ErrorResultException.class) public ResultJson deleteExtension(String namespaceName, String extensionName, UserData admin) throws ErrorResultException { @@ -183,7 +186,11 @@ public ResultJson editNamespaceMember(String namespaceName, String userName, Str @Transactional(rollbackOn = ErrorResultException.class) public ResultJson createNamespace(NamespaceJson json) { - validateNamespace(json.name); + var namespaceIssue = validator.validateNamespace(json.name); + if (namespaceIssue.isPresent()) { + throw new ErrorResultException(namespaceIssue.get().toString()); + } + var namespace = repositories.findNamespace(json.name); if (namespace != null) { throw new ErrorResultException("Namespace already exists: " + namespace.getName()); @@ -194,12 +201,11 @@ public ResultJson createNamespace(NamespaceJson json) { return ResultJson.success("Created namespace " + namespace.getName()); } - @Transactional public void changeNamespace(ChangeNamespaceJson json) { - if(Strings.isNullOrEmpty(json.oldNamespace)) { + if (Strings.isNullOrEmpty(json.oldNamespace)) { throw new ErrorResultException("Old namespace must have a value"); } - if(Strings.isNullOrEmpty(json.newNamespace)) { + if (Strings.isNullOrEmpty(json.newNamespace)) { throw new ErrorResultException("New namespace must have a value"); } @@ -209,40 +215,11 @@ public void changeNamespace(ChangeNamespaceJson json) { } var newNamespace = repositories.findNamespace(json.newNamespace); - if(newNamespace != null && !json.mergeIfNewNamespaceAlreadyExists) { + if (newNamespace != null && !json.mergeIfNewNamespaceAlreadyExists) { throw new ErrorResultException("New namespace already exists: " + json.newNamespace); } - if(newNamespace == null) { - validateNamespace(json.newNamespace); - newNamespace = new Namespace(); - newNamespace.setName(json.newNamespace); - entityManager.persist(newNamespace); - } - - var extensions = repositories.findExtensions(oldNamespace); - for(var extension : extensions) { - cache.evictExtensionJsons(extension); - cache.evictLatestExtensionVersion(extension); - extension.setNamespace(newNamespace); - } - var memberships = repositories.findMemberships(oldNamespace); - for(var membership : memberships) { - membership.setNamespace(newNamespace); - } - - if(json.removeOldNamespace) { - entityManager.remove(oldNamespace); - } - - search.updateSearchEntries(extensions.toList()); - } - - private void validateNamespace(String namespace) { - var namespaceIssue = validator.validateNamespace(namespace); - if (namespaceIssue.isPresent()) { - throw new ErrorResultException(namespaceIssue.get().toString()); - } + scheduler.enqueue(new ChangeNamespaceJobRequest(json)); } public UserPublishInfoJson getUserPublishInfo(String provider, String loginName) { diff --git a/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceJobRequest.java b/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceJobRequest.java new file mode 100644 index 000000000..fcc3d4093 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceJobRequest.java @@ -0,0 +1,53 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx; + +import org.eclipse.openvsx.json.ChangeNamespaceJson; +import org.jobrunr.jobs.lambdas.JobRequest; +import org.jobrunr.jobs.lambdas.JobRequestHandler; + +import java.util.Objects; + +public class ChangeNamespaceJobRequest implements JobRequest { + + private ChangeNamespaceJson data; + + public ChangeNamespaceJobRequest() {} + + public ChangeNamespaceJobRequest(ChangeNamespaceJson data) { + this.data = data; + } + + @Override + public Class getJobRequestHandler() { + return ChangeNamespaceJobRequestHandler.class; + } + + public ChangeNamespaceJson getData() { + return data; + } + + public void setData(ChangeNamespaceJson data) { + this.data = data; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ChangeNamespaceJobRequest that = (ChangeNamespaceJobRequest) o; + return Objects.equals(data, that.data); + } + + @Override + public int hashCode() { + return Objects.hash(data); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceJobRequestHandler.java new file mode 100644 index 000000000..e37e06a7f --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceJobRequestHandler.java @@ -0,0 +1,195 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx; + +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.entities.ExtensionVersion; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.entities.Namespace; +import org.eclipse.openvsx.repositories.RepositoryService; +import org.eclipse.openvsx.storage.StorageUtilService; +import org.eclipse.openvsx.util.ErrorResultException; +import org.jobrunr.jobs.lambdas.JobRequestHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.util.Pair; +import org.springframework.data.util.Streamable; +import org.springframework.stereotype.Component; + +import java.util.*; +import java.util.stream.Collectors; + +@Component +public class ChangeNamespaceJobRequestHandler implements JobRequestHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(ChangeNamespaceJobRequestHandler.class); + + private static final Map LOCKS; + + static { + var MAX_SIZE = 100; + LOCKS = Collections.synchronizedMap(new LinkedHashMap<>(MAX_SIZE) { + protected boolean removeEldestEntry(Map.Entry eldest){ + return size() > MAX_SIZE; + } + }); + } + + @Autowired + ExtensionValidator validator; + + @Autowired + RepositoryService repositories; + + @Autowired + StorageUtilService storageUtil; + + @Autowired + ChangeNamespaceService service; + + @Override + public void run(ChangeNamespaceJobRequest jobRequest) throws Exception { + var oldNamespace = jobRequest.getData().oldNamespace; + synchronized (LOCKS.computeIfAbsent(oldNamespace, key -> new Object())) { + execute(jobRequest); + } + } + + private void execute(ChangeNamespaceJobRequest jobRequest) { + var json = jobRequest.getData(); + LOGGER.info(">> Change namespace from {} to {}", json.oldNamespace, json.newNamespace); + var oldNamespace = repositories.findNamespace(json.oldNamespace); + if(oldNamespace == null) { + return; + } + + var oldResources = repositories.findFileResources(oldNamespace); + var newNamespaceOptional = Optional.ofNullable(repositories.findNamespace(json.newNamespace)); + var createNewNamespace = newNamespaceOptional.isEmpty(); + var newNamespace = newNamespaceOptional.orElseGet(() -> { + validateNamespace(json.newNamespace); + var namespace = new Namespace(); + namespace.setName(json.newNamespace); + return namespace; + }); + + var copyResources = oldResources.stream() + .findFirst() + .map(storageUtil::shouldStoreExternally) + .orElse(false); + + List> pairs = null; + List updatedResources; + if(copyResources) { + pairs = copyResources(oldResources, newNamespace); + storageUtil.copyFiles(pairs); + updatedResources = pairs.stream() + .filter(pair -> pair.getFirst().getType().equals(FileResource.DOWNLOAD)) + .map(pair -> { + var oldResource = pair.getFirst(); + var newResource = pair.getSecond(); + oldResource.setName(newResource.getName()); + return oldResource; + }) + .collect(Collectors.toList()); + } else { + updatedResources = oldResources + .filter(resource -> resource.getType().equals(FileResource.DOWNLOAD)) + .map(resource -> { + resource.setName(newResourceName(newNamespace, resource)); + return resource; + }) + .toList(); + } + + service.changeNamespaceInDatabase(newNamespace, oldNamespace, updatedResources, createNewNamespace, json.removeOldNamespace); + if(copyResources) { + // remove the old resources from external storage + pairs.stream() + .map(Pair::getFirst) + .forEach(storageUtil::removeFile); + } + LOGGER.info("<< Changed namespace from {} to {}", json.oldNamespace, json.newNamespace); + } + + private void validateNamespace(String namespace) { + var namespaceIssue = validator.validateNamespace(namespace); + if (namespaceIssue.isPresent()) { + throw new ErrorResultException(namespaceIssue.get().toString()); + } + } + + private List> copyResources(Streamable resources, Namespace newNamespace) { + var extVersions = resources.stream() + .map(FileResource::getExtension) + .collect(Collectors.toMap(ExtensionVersion::getId, ev -> ev, (ev1, ev2) -> ev1)); + + var extensions = extVersions.values().stream() + .map(ExtensionVersion::getExtension) + .collect(Collectors.groupingBy(Extension::getId)) + .entrySet().stream() + .map(entry -> { + var extension = entry.getValue().get(0); + var newExtension = new Extension(); + newExtension.setId(extension.getId()); + newExtension.setName(extension.getName()); + newExtension.setNamespace(newNamespace); + return new AbstractMap.SimpleEntry<>(entry.getKey(), newExtension); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + for(var key : extVersions.keySet()) { + var extVersion = extVersions.get(key); + var newExtVersion = new ExtensionVersion(); + newExtVersion.setId(extVersion.getId()); + newExtVersion.setExtension(extensions.get(extVersion.getExtension().getId())); + newExtVersion.setVersion(extVersion.getVersion()); + newExtVersion.setTargetPlatform(extVersion.getTargetPlatform()); + extVersions.put(key, newExtVersion); + } + + return resources.stream() + .map(resource -> { + var newExtVersion = extVersions.get(resource.getExtension().getId()); + var newResource = new FileResource(); + newResource.setId(resource.getId()); + newResource.setExtension(newExtVersion); + newResource.setType(resource.getType()); + newResource.setStorageType(resource.getStorageType()); + var newResourceName = resource.getType().equals(FileResource.DOWNLOAD) + ? newResourceName(newNamespace, resource) + : resource.getName(); + + newResource.setName(newResourceName); + return Pair.of(resource, newResource); + }) + .collect(Collectors.toList()); + } + + private String newResourceName(Namespace newNamespace, FileResource resource) { + var extVersion = resource.getExtension(); + var extension = extVersion.getExtension(); + + var newExtension = new Extension(); + newExtension.setNamespace(newNamespace); + newExtension.setName(extension.getName()); + + var newExtVersion = new ExtensionVersion(); + newExtVersion.setVersion(extVersion.getVersion()); + newExtVersion.setTargetPlatform(extVersion.getTargetPlatform()); + newExtVersion.setExtension(newExtension); + try(var processor = new ExtensionProcessor(null)) { + var newResourceName = processor.getBinaryName(newExtVersion); + LOGGER.info("newResourceName: {}", newResourceName); + return newResourceName; + } + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceService.java b/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceService.java new file mode 100644 index 000000000..d4c44c876 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceService.java @@ -0,0 +1,86 @@ +/** ****************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx; + +import org.eclipse.openvsx.cache.CacheService; +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.entities.Namespace; +import org.eclipse.openvsx.repositories.RepositoryService; +import org.eclipse.openvsx.search.SearchUtilService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.util.Streamable; +import org.springframework.stereotype.Component; + +import javax.persistence.EntityManager; +import javax.transaction.Transactional; +import java.util.List; + +@Component +public class ChangeNamespaceService { + + @Autowired + RepositoryService repositories; + + @Autowired + EntityManager entityManager; + + @Autowired + CacheService cache; + + @Autowired + SearchUtilService search; + + @Transactional + public void changeNamespaceInDatabase( + Namespace newNamespace, + Namespace oldNamespace, + List updatedResources, + boolean createNewNamespace, + boolean removeOldNamespace + ) { + var extensions = repositories.findExtensions(oldNamespace); + for(var extension : extensions) { + cache.evictExtensionJsons(extension); + cache.evictLatestExtensionVersion(extension); + } + + if(createNewNamespace) { + entityManager.persist(newNamespace); + } else { + newNamespace = entityManager.merge(newNamespace); + } + + changeExtensionNamespace(extensions, newNamespace); + changeMembershipNamespace(oldNamespace, newNamespace); + updatedResources.forEach(entityManager::merge); + + if(removeOldNamespace) { + oldNamespace = entityManager.merge(oldNamespace); + entityManager.remove(oldNamespace); + } + + search.updateSearchEntries(extensions.toList()); + } + + private void changeExtensionNamespace(Streamable extensions, Namespace newNamespace) { + for(var extension : extensions) { + extension = entityManager.merge(extension); + extension.setNamespace(newNamespace); + } + } + + private void changeMembershipNamespace(Namespace oldNamespace, Namespace newNamespace) { + var memberships = repositories.findMemberships(oldNamespace); + for(var membership : memberships) { + membership.setNamespace(newNamespace); + } + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java b/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java index 2a8bc5826..c92c9dc5d 100644 --- a/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java +++ b/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java @@ -340,7 +340,7 @@ public FileResource getBinary(ExtensionVersion extVersion) { return binary; } - private String getBinaryName(ExtensionVersion extVersion) { + public String getBinaryName(ExtensionVersion extVersion) { var extension = extVersion.getExtension(); var namespace = extension.getNamespace(); var resourceName = namespace.getName() + "." + extension.getName() + "-" + extVersion.getVersion(); diff --git a/server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java b/server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java index 20cd41469..6823a5a36 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java @@ -11,6 +11,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.Objects; + /** * Used to change a namespace */ @@ -20,4 +22,17 @@ public class ChangeNamespaceJson { public String newNamespace; public boolean removeOldNamespace; public boolean mergeIfNewNamespaceAlreadyExists; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ChangeNamespaceJson that = (ChangeNamespaceJson) o; + return removeOldNamespace == that.removeOldNamespace && mergeIfNewNamespaceAlreadyExists == that.mergeIfNewNamespaceAlreadyExists && Objects.equals(oldNamespace, that.oldNamespace) && Objects.equals(newNamespace, that.newNamespace); + } + + @Override + public int hashCode() { + return Objects.hash(oldNamespace, newNamespace, removeOldNamespace, mergeIfNewNamespaceAlreadyExists); + } } diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceRepository.java b/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceRepository.java index a0a19a4d7..467cef523 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceRepository.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/FileResourceRepository.java @@ -9,6 +9,7 @@ ********************************************************************************/ package org.eclipse.openvsx.repositories; +import org.eclipse.openvsx.entities.Namespace; import org.springframework.data.repository.Repository; import org.springframework.data.util.Streamable; @@ -21,6 +22,8 @@ public interface FileResourceRepository extends Repository { Streamable findByExtension(ExtensionVersion extVersion); + Streamable findByExtensionExtensionNamespace(Namespace namespace); + Streamable findByStorageType(String storageType); FileResource findFirstByExtensionAndNameIgnoreCaseOrderByType(ExtensionVersion extVersion, String name); diff --git a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java index eb52ec304..0f642a358 100644 --- a/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java +++ b/server/src/main/java/org/eclipse/openvsx/repositories/RepositoryService.java @@ -421,4 +421,8 @@ public double getAverageReviewRating() { public Double getAverageReviewRating(Extension extension) { return extensionReviewRepo.averageRatingAndActiveTrue(extension); } + + public Streamable findFileResources(Namespace namespace) { + return fileResourceRepo.findByExtensionExtensionNamespace(namespace); + } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java index b66ccbbce..11eb9e845 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java @@ -9,10 +9,13 @@ ********************************************************************************/ package org.eclipse.openvsx.storage; +import com.azure.core.util.polling.SyncPoller; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobContainerClientBuilder; +import com.azure.storage.blob.models.BlobCopyInfo; import com.azure.storage.blob.models.BlobHttpHeaders; import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.CopyStatusType; import com.google.common.base.Strings; import org.apache.commons.lang3.ArrayUtils; import org.eclipse.openvsx.entities.FileResource; @@ -20,6 +23,7 @@ import org.eclipse.openvsx.util.TargetPlatform; import org.eclipse.openvsx.util.UrlUtil; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.util.Pair; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; @@ -28,6 +32,10 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; @Component public class AzureBlobStorageService implements IStorageService { @@ -201,4 +209,23 @@ public Path downloadNamespaceLogo(Namespace namespace) { throw new RuntimeException(e); } } + + @Override + public void copyFiles(List> pairs) { + var copyOperations = new ArrayList>(); + for(var pair : pairs) { + var oldLocation = getLocation(pair.getFirst()).toString(); + var newBlobName = getBlobName(pair.getSecond()); + var poller = getContainerClient().getBlobClient(newBlobName) + .beginCopy(oldLocation, Duration.of(1, ChronoUnit.SECONDS)); + + copyOperations.add(poller); + } + for(var poller : copyOperations) { + var response = poller.waitForCompletion(); + if(response.getValue().getCopyStatus() != CopyStatusType.SUCCESS) { + throw new RuntimeException(response.getValue().getError()); + } + } + } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java index 7f78b22a0..7879946b8 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java @@ -20,6 +20,7 @@ import org.eclipse.openvsx.util.TargetPlatform; import org.eclipse.openvsx.util.UrlUtil; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.util.Pair; import org.springframework.stereotype.Component; import java.io.FileOutputStream; @@ -28,6 +29,7 @@ import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; @Component public class GoogleCloudStorageService implements IStorageService { @@ -206,4 +208,18 @@ public Path downloadNamespaceLogo(Namespace namespace) { return logoFile; } + + @Override + public void copyFiles(List> pairs) { + for(var pair : pairs) { + var source = getObjectId(pair.getFirst()); + var target = getObjectId(pair.getSecond()); + var request = new Storage.CopyRequest.Builder() + .setSource(BlobId.of(bucketId, source)) + .setTarget(BlobId.of(bucketId, target)) + .build(); + + getStorage().copy(request).getResult(); + } + } } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java index 991da607f..36329cb46 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/IStorageService.java @@ -11,9 +11,11 @@ import java.net.URI; import java.nio.file.Path; +import java.util.List; import org.eclipse.openvsx.entities.FileResource; import org.eclipse.openvsx.entities.Namespace; +import org.springframework.data.util.Pair; public interface IStorageService { @@ -58,4 +60,6 @@ public interface IStorageService { URI getNamespaceLogoLocation(Namespace namespace); Path downloadNamespaceLogo(Namespace namespace); + + void copyFiles(List> pairs); } \ No newline at end of file diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java index 7988afba9..1affe00cc 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java @@ -22,6 +22,7 @@ import org.eclipse.openvsx.util.UrlUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.util.Pair; import org.springframework.http.CacheControl; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; @@ -342,4 +343,16 @@ public ResponseEntity getNamespaceLogo(Namespace namespace) { .build(); } } + + @Override + public void copyFiles(List> pairs) { + switch (pairs.get(0).getFirst().getStorageType()) { + case STORAGE_GOOGLE: + googleStorage.copyFiles(pairs); + break; + case STORAGE_AZURE: + azureStorage.copyFiles(pairs); + break; + } + } } \ No newline at end of file diff --git a/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java b/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java index adfbe30c4..3786f324c 100644 --- a/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/AdminAPITest.java @@ -29,6 +29,7 @@ import org.eclipse.openvsx.storage.StorageUtilService; import org.eclipse.openvsx.util.TargetPlatform; import org.eclipse.openvsx.util.VersionService; +import org.jobrunr.scheduling.JobRequestScheduler; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; @@ -74,6 +75,9 @@ public class AdminAPITest { @SpyBean UserService users; + @MockBean + JobRequestScheduler scheduler; + @MockBean RepositoryService repositories; @@ -1031,12 +1035,6 @@ public void testChangeNamespace() throws Exception { bar.setName("bar"); Mockito.when(repositories.findNamespace(bar.getName())).thenReturn(null); - var extension = Mockito.mock(Extension.class); - Mockito.when(repositories.findExtensions(foo)).thenReturn(Streamable.of(extension)); - - var membership = Mockito.mock(NamespaceMembership.class); - Mockito.when(repositories.findMemberships(foo)).thenReturn(Streamable.of(membership)); - var content = "{" + "\"oldNamespace\": \"foo\", " + "\"newNamespace\": \"bar\", " + @@ -1044,15 +1042,20 @@ public void testChangeNamespace() throws Exception { "\"mergeIfNewNamespaceAlreadyExists\": true" + "}"; + var json = new ChangeNamespaceJson(); + json.oldNamespace = "foo"; + json.newNamespace = "bar"; + json.removeOldNamespace = false; + json.mergeIfNewNamespaceAlreadyExists = true; + mockMvc.perform(post("/admin/change-namespace") .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) .with(csrf().asHeader()) .content(content) .contentType(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()) - .andExpect(content().json(successJson("Changed namespace foo to bar"))) - .andExpect(result -> Mockito.verify(extension).setNamespace(bar)) - .andExpect(result -> Mockito.verify(membership).setNamespace(bar)); + .andExpect(content().json(successJson("Scheduled namespace change from 'foo' to 'bar'.\nIt can take 15 minutes to a couple hours for the change to become visible."))) + .andExpect(result -> Mockito.verify(scheduler).enqueue(new ChangeNamespaceJobRequest(json))); } @Test @@ -1156,77 +1159,6 @@ public void testChangeNamespaceNewNamespaceEmpty() throws Exception { .andExpect(content().json(errorJson("New namespace must have a value"))); } - @Test - public void testChangeNamespaceNewNamespaceAlreadyExists() throws Exception { - mockAdminUser(); - var foo = new Namespace(); - foo.setName("foo"); - Mockito.when(repositories.findNamespace(foo.getName())).thenReturn(foo); - - var bar = new Namespace(); - bar.setName("bar"); - Mockito.when(repositories.findNamespace(bar.getName())).thenReturn(bar); - - var extension = Mockito.mock(Extension.class); - Mockito.when(repositories.findExtensions(foo)).thenReturn(Streamable.of(extension)); - - var membership = Mockito.mock(NamespaceMembership.class); - Mockito.when(repositories.findMemberships(foo)).thenReturn(Streamable.of(membership)); - - var content = "{" + - "\"oldNamespace\": \"foo\", " + - "\"newNamespace\": \"bar\", " + - "\"removeOldNamespace\": false, " + - "\"mergeIfNewNamespaceAlreadyExists\": true" + - "}"; - - mockMvc.perform(post("/admin/change-namespace") - .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) - .with(csrf().asHeader()) - .content(content) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(content().json(successJson("Changed namespace foo to bar"))) - .andExpect(result -> Mockito.verify(extension).setNamespace(bar)) - .andExpect(result -> Mockito.verify(membership).setNamespace(bar)); - } - - @Test - public void testChangeNamespaceRemoveOldNamespace() throws Exception { - mockAdminUser(); - var foo = new Namespace(); - foo.setName("foo"); - Mockito.when(repositories.findNamespace(foo.getName())).thenReturn(foo); - - var bar = new Namespace(); - bar.setName("bar"); - Mockito.when(repositories.findNamespace(bar.getName())).thenReturn(bar); - - var extension = Mockito.mock(Extension.class); - Mockito.when(repositories.findExtensions(foo)).thenReturn(Streamable.of(extension)); - - var membership = Mockito.mock(NamespaceMembership.class); - Mockito.when(repositories.findMemberships(foo)).thenReturn(Streamable.of(membership)); - - var content = "{" + - "\"oldNamespace\": \"foo\", " + - "\"newNamespace\": \"bar\", " + - "\"removeOldNamespace\": true, " + - "\"mergeIfNewNamespaceAlreadyExists\": true" + - "}"; - - mockMvc.perform(post("/admin/change-namespace") - .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) - .with(csrf().asHeader()) - .content(content) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(content().json(successJson("Changed namespace foo to bar"))) - .andExpect(result -> Mockito.verify(extension).setNamespace(bar)) - .andExpect(result -> Mockito.verify(membership).setNamespace(bar)) - .andExpect(result -> Mockito.verify(entityManager).remove(foo)); - } - @Test public void testChangeNamespaceAbortOnNewNamespaceExists() throws Exception { mockAdminUser(); @@ -1254,41 +1186,6 @@ public void testChangeNamespaceAbortOnNewNamespaceExists() throws Exception { .andExpect(content().json(errorJson("New namespace already exists: bar"))); } - @Test - public void testChangeNamespaceAbortNewNamespaceDoesNotExist() throws Exception { - mockAdminUser(); - var foo = new Namespace(); - foo.setName("foo"); - Mockito.when(repositories.findNamespace(foo.getName())).thenReturn(foo); - - var bar = new Namespace(); - bar.setName("bar"); - Mockito.when(repositories.findNamespace(bar.getName())).thenReturn(null); - - var extension = Mockito.mock(Extension.class); - Mockito.when(repositories.findExtensions(foo)).thenReturn(Streamable.of(extension)); - - var membership = Mockito.mock(NamespaceMembership.class); - Mockito.when(repositories.findMemberships(foo)).thenReturn(Streamable.of(membership)); - - var content = "{" + - "\"oldNamespace\": \"foo\", " + - "\"newNamespace\": \"bar\", " + - "\"removeOldNamespace\": false, " + - "\"mergeIfNewNamespaceAlreadyExists\": false" + - "}"; - - mockMvc.perform(post("/admin/change-namespace") - .with(user("admin_user").authorities(new SimpleGrantedAuthority(("ROLE_ADMIN")))) - .with(csrf().asHeader()) - .content(content) - .contentType(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) - .andExpect(content().json(successJson("Changed namespace foo to bar"))) - .andExpect(result -> Mockito.verify(extension).setNamespace(bar)) - .andExpect(result -> Mockito.verify(membership).setNamespace(bar)); - } - //---------- UTILITY ----------// private PersonalAccessToken mockAdminToken() { diff --git a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java index a4394d1d4..061eb7a3d 100644 --- a/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java +++ b/server/src/test/java/org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.java @@ -166,7 +166,8 @@ void testExecuteQueries() { () -> repositories.findActiveExtensionVersionsByNamespacePublicId("targetPlatform", "namespacePublicId"), () -> repositories.findAllNotMatchingByExtensionId(STRING_LIST), () -> repositories.getAverageReviewRating(null), - () -> repositories.getAverageReviewRating() + () -> repositories.getAverageReviewRating(), + () -> repositories.findFileResources(null) ); // check that we did not miss anything diff --git a/webui/src/components/info-dialog.tsx b/webui/src/components/info-dialog.tsx new file mode 100644 index 000000000..8d5fe5fd3 --- /dev/null +++ b/webui/src/components/info-dialog.tsx @@ -0,0 +1,70 @@ +/******************************************************************************** + * Copyright (c) 2023 Precies. Software Ltd and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ + +import * as React from 'react'; +import { + Dialog, DialogTitle, DialogContent, Button, DialogContentText, DialogActions, + Theme, createStyles, withStyles, WithStyles +} from '@material-ui/core'; +import { MainContext } from '../context'; + +const dialogStyles = (theme: Theme) => createStyles({ + dialogContent: { + color: theme.palette.text.primary + } +}); + +export class InfoDialogComponent extends React.Component { + + static contextType = MainContext; + declare context: MainContext; + + handleEnter = (event: KeyboardEvent): void => { + if (event.code === 'Enter') { + this.props.handleCloseDialog(); + } + }; + + componentDidMount(): void { + document.addEventListener('keydown', this.handleEnter); + } + + componentWillUnmount(): void { + document.removeEventListener('keydown', this.handleEnter); + } + + render(): React.ReactNode { + return + Info + + + {this.props.infoMessage} + + + + + + ; + } +} + +export namespace InfoDialogComponent { + export interface Props extends WithStyles { + infoMessage: string; + isInfoDialogOpen: boolean; + handleCloseDialog: () => void; + } +} + +export const InfoDialog = withStyles(dialogStyles)(InfoDialogComponent); diff --git a/webui/src/pages/admin-dashboard/namespace-admin.tsx b/webui/src/pages/admin-dashboard/namespace-admin.tsx index baf1c7ce6..af9871c4f 100644 --- a/webui/src/pages/admin-dashboard/namespace-admin.tsx +++ b/webui/src/pages/admin-dashboard/namespace-admin.tsx @@ -89,7 +89,6 @@ export const NamespaceAdmin: FunctionComponent = props => { setLoadingState={setLoading} namespace={currentNamespace} filterUsers={() => true} - onNamespaceChange={fetchNamespace} fixSelf={false} /> diff --git a/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx b/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx index d3aeb33be..7463b14f4 100644 --- a/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx +++ b/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx @@ -15,10 +15,11 @@ import { ButtonWithProgress } from '../../components/button-with-progress'; import { Namespace, isError } from '../../extension-registry-types'; import { MainContext } from '../../context'; + import { InfoDialog } from '../../components/info-dialog'; export interface NamespaceChangeDialogProps { open: boolean; - onClose: (newNamespaceName?: string) => void; + onClose: () => void; namespace: Namespace; setLoadingState: (loading: boolean) => void; } @@ -30,6 +31,8 @@ const [newNamespace, setNewNamespace] = useState(''); const [removeOldNamespace, setRemoveOldNamespace] = useState(false); const [mergeIfNewNamespaceAlreadyExists, setMergeIfNewNamespaceAlreadyExists] = useState(false); + const [infoDialogIsOpen, setInfoDialogIsOpen] = useState(false); + const [infoDialogMessage, setInfoDialogMessage] = useState(''); const abortController = new AbortController(); useEffect(() => { @@ -69,7 +72,8 @@ } props.setLoadingState(false); setWorking(false); - props.onClose(newNamespace); + setInfoDialogIsOpen(true); + setInfoDialogMessage(result.success); } catch (err) { props.setLoadingState(false); setWorking(false); @@ -116,5 +120,6 @@
+ {props.onClose(); setInfoDialogIsOpen(false);}}/> ; }; \ No newline at end of file diff --git a/webui/src/pages/user/user-settings-namespace-detail.tsx b/webui/src/pages/user/user-settings-namespace-detail.tsx index 66db65907..0dbc0ceab 100644 --- a/webui/src/pages/user/user-settings-namespace-detail.tsx +++ b/webui/src/pages/user/user-settings-namespace-detail.tsx @@ -86,11 +86,8 @@ const useStyles = makeStyles((theme) => ({ export const NamespaceDetailComponent: FunctionComponent = props => { const classes = useStyles(); const [changeDialogIsOpen, setChangeDialogIsOpen] = useState(false); - const handleCloseChangeDialog = async (newNamespaceName?: string) => { + const handleCloseChangeDialog = async () => { setChangeDialogIsOpen(false); - if (newNamespaceName && props.onNamespaceChange) { - props.onNamespaceChange(newNamespaceName); - } }; const handleOpenChangeDialog = () => { setChangeDialogIsOpen(true); @@ -157,7 +154,6 @@ export namespace NamespaceDetailComponent { export interface Props extends RouteComponentProps { namespace: Namespace; filterUsers: (user: UserData) => boolean; - onNamespaceChange?: (newNamespaceName: string) => void; fixSelf: boolean; setLoadingState: (loading: boolean) => void; namespaceAccessUrl?: string; From fd51cc06687639415ed7f47fa3bc07b4799c3484 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Tue, 21 Mar 2023 22:10:02 +0200 Subject: [PATCH 34/45] Download count update partition ElasticSearch search entries update --- .../migration/OrphanNamespaceMigration.java | 12 ++++----- .../openvsx/search/DatabaseSearchService.java | 8 ++++++ .../openvsx/search/ElasticSearchService.java | 15 ++++++----- .../openvsx/search/ISearchService.java | 2 ++ .../openvsx/search/RelevanceService.java | 26 +++++++++++++------ .../openvsx/search/SearchUtilService.java | 5 ++++ .../storage/AzureDownloadCountProcessor.java | 15 ++++++++--- 7 files changed, 59 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/migration/OrphanNamespaceMigration.java b/server/src/main/java/org/eclipse/openvsx/migration/OrphanNamespaceMigration.java index 8d16d5132..50492357e 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/OrphanNamespaceMigration.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/OrphanNamespaceMigration.java @@ -9,21 +9,18 @@ ********************************************************************************/ package org.eclipse.openvsx.migration; -import java.util.LinkedHashSet; - -import javax.persistence.EntityManager; -import javax.transaction.Transactional; - import org.eclipse.openvsx.entities.NamespaceMembership; import org.eclipse.openvsx.entities.UserData; import org.eclipse.openvsx.repositories.RepositoryService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.event.ApplicationStartedEvent; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; +import javax.persistence.EntityManager; +import javax.transaction.Transactional; +import java.util.LinkedHashSet; + @Component public class OrphanNamespaceMigration { @@ -35,6 +32,7 @@ public class OrphanNamespaceMigration { @Autowired RepositoryService repositories; + @Transactional public void fixOrphanNamespaces() { int[] count = new int[3]; repositories.findOrphanNamespaces().forEach(namespace -> { diff --git a/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java b/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java index 0bdce690b..704d87e75 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java @@ -19,6 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.elasticsearch.core.*; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.repositories.RepositoryService; @@ -160,6 +161,13 @@ public void updateSearchIndex(boolean clear) { } + @Override + @Async + @CacheEvict(value = CACHE_DATABASE_SEARCH, allEntries = true) + public void updateSearchEntriesAsync(List extensions) { + + } + @Override @CacheEvict(value = CACHE_DATABASE_SEARCH, allEntries = true) public void updateSearchEntries(List extensions) { diff --git a/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java b/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java index 557e79e50..9e9e5abd2 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/ElasticSearchService.java @@ -43,9 +43,9 @@ import org.springframework.data.elasticsearch.core.query.IndexQueryBuilder; import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder; import org.springframework.retry.annotation.Retryable; +import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StopWatch; import static org.eclipse.openvsx.cache.CacheService.CACHE_AVERAGE_REVIEW_RATING; @@ -92,7 +92,6 @@ public boolean isEnabled() { * not exist yet, it is created and initialized. Otherwise nothing happens. */ @EventListener - @Transactional(readOnly = true) @Retryable(DataAccessResourceFailureException.class) @CacheEvict(value = CACHE_AVERAGE_REVIEW_RATING, allEntries = true) public void initSearchIndex(ApplicationStartedEvent event) { @@ -112,7 +111,6 @@ public void initSearchIndex(ApplicationStartedEvent event) { * timestamps in relation to the current time or the extension rating. */ @Scheduled(cron = "0 0 4 * * *", zone = "UTC") - @Transactional(readOnly = true) @Retryable(DataAccessResourceFailureException.class) @CacheEvict(value = CACHE_AVERAGE_REVIEW_RATING, allEntries = true) public void updateSearchIndex() { @@ -135,7 +133,6 @@ public void updateSearchIndex() { * In any case, this method scans all extensions in the database and indexes their * relevant metadata. */ - @Transactional @Retryable(DataAccessResourceFailureException.class) public void updateSearchIndex(boolean clear) { var locked = false; @@ -164,7 +161,7 @@ public void updateSearchIndex(boolean clear) { var stats = new SearchStats(repositories); var indexQueries = allExtensions.map(extension -> new IndexQueryBuilder() - .withObject(relevanceService.toSearchEntry(extension, stats)) + .withObject(relevanceService.toSearchEntryTrxn(extension, stats)) .build() ).toList(); @@ -181,6 +178,12 @@ public void updateSearchIndex(boolean clear) { } } + @Async + @Retryable(DataAccessResourceFailureException.class) + public void updateSearchEntriesAsync(List extensions) { + updateSearchEntries(extensions); + } + @Retryable(DataAccessResourceFailureException.class) public void updateSearchEntries(List extensions) { if (!isEnabled() || extensions.isEmpty()) { @@ -192,7 +195,7 @@ public void updateSearchEntries(List extensions) { var stats = new SearchStats(repositories); var indexQueries = extensions.stream().map(extension -> new IndexQueryBuilder() - .withObject(relevanceService.toSearchEntry(extension, stats)) + .withObject(relevanceService.toSearchEntryTrxn(extension, stats)) .build() ).collect(Collectors.toList()); searchOperations.bulkIndex(indexQueries, indexOps.getIndexCoordinates()); diff --git a/server/src/main/java/org/eclipse/openvsx/search/ISearchService.java b/server/src/main/java/org/eclipse/openvsx/search/ISearchService.java index 3e0d2c2c2..1e1185565 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/ISearchService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/ISearchService.java @@ -44,6 +44,8 @@ public interface ISearchService { */ void updateSearchEntries(List extensions); + void updateSearchEntriesAsync(List extensions); + /** * The given extension has been added to the registry, we need to refresh the search index. */ diff --git a/server/src/main/java/org/eclipse/openvsx/search/RelevanceService.java b/server/src/main/java/org/eclipse/openvsx/search/RelevanceService.java index f538e5e8e..d7846dc45 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/RelevanceService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/RelevanceService.java @@ -10,14 +10,8 @@ package org.eclipse.openvsx.search; -import java.time.Duration; -import java.time.LocalDateTime; -import java.util.Optional; -import javax.annotation.PostConstruct; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; - import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.NamespaceMembership; @@ -27,8 +21,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.persistence.EntityManager; +import javax.transaction.Transactional; +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Optional; /** * Provides relevance for a given extension @@ -56,6 +57,9 @@ public class RelevanceService { @Value("${ovsx.elasticsearch.relevance.unverified:-1.0}") double deprecatedElasticSearchUnverifiedRelevance; + @Autowired + EntityManager entityManager; + @Autowired RepositoryService repositories; @@ -82,7 +86,13 @@ void init() { } } - protected ExtensionSearch toSearchEntry(Extension extension, SearchStats stats) { + @Transactional + public ExtensionSearch toSearchEntryTrxn(Extension extension, SearchStats stats) { + extension = entityManager.merge(extension); + return toSearchEntry(extension, stats); + } + + public ExtensionSearch toSearchEntry(Extension extension, SearchStats stats) { var latest = versions.getLatest(extension, null, false, true); var entry = extension.toSearch(latest); entry.rating = calculateRating(extension, stats); diff --git a/server/src/main/java/org/eclipse/openvsx/search/SearchUtilService.java b/server/src/main/java/org/eclipse/openvsx/search/SearchUtilService.java index 3ba4fc3ba..fde318695 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/SearchUtilService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/SearchUtilService.java @@ -67,6 +67,11 @@ public void updateSearchEntries(List extensions) { getImplementation().updateSearchEntries(extensions); } + @Override + public void updateSearchEntriesAsync(List extensions) { + getImplementation().updateSearchEntriesAsync(extensions); + } + @Override public void updateSearchEntry(Extension extension) { getImplementation().updateSearchEntry(extension); diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountProcessor.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountProcessor.java index 04da8d002..725d030c6 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountProcessor.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureDownloadCountProcessor.java @@ -9,10 +9,13 @@ ********************************************************************************/ package org.eclipse.openvsx.storage; +import com.google.common.collect.Lists; import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.entities.*; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.SearchUtilService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -27,6 +30,8 @@ @Component public class AzureDownloadCountProcessor { + protected final Logger logger = LoggerFactory.getLogger(AzureDownloadCountProcessor.class); + @Autowired EntityManager entityManager; @@ -90,14 +95,18 @@ public void evictCaches(List extensions) { }); } - @Transactional //needs transaction for lazy-loading versions public void updateSearchEntries(List extensions) { + logger.info(">> updateSearchEntries"); var activeExtensions = extensions.stream() .filter(Extension::isActive) - .map(entityManager::merge) .collect(Collectors.toList()); - search.updateSearchEntries(activeExtensions); + logger.info("total active extensions: {}", activeExtensions.size()); + var parts = Lists.partition(activeExtensions, 100); + logger.info("partitions: {} | partition size: 100", parts.size()); + + parts.forEach(search::updateSearchEntriesAsync); + logger.info("<< updateSearchEntries"); } public List processedItems(List blobNames) { From e590b64cf0e1183a3053a570743564857385501c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Mar 2023 11:28:35 +0000 Subject: [PATCH 35/45] Bump webpack from 5.75.0 to 5.76.0 in /webui Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.0. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.0) --- updated-dependencies: - dependency-name: webpack dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- webui/yarn.lock | 72 ++++++------------------------------------------- 1 file changed, 8 insertions(+), 64 deletions(-) diff --git a/webui/yarn.lock b/webui/yarn.lock index 20e295bba..4700a0393 100644 --- a/webui/yarn.lock +++ b/webui/yarn.lock @@ -379,11 +379,6 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== -"@types/json-schema@^7.0.6": - version "7.0.6" - resolved "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.6.tgz" - integrity sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw== - "@types/json5@^0.0.29": version "0.0.29" resolved "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz" @@ -801,16 +796,11 @@ acorn-jsx@^5.3.2: resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== -acorn@^8.5.0, acorn@^8.7.1: +acorn@^8.5.0, acorn@^8.7.1, acorn@^8.8.0: version "8.8.2" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== -acorn@^8.8.0: - version "8.8.1" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.1.tgz#0a3f9cbecc4ec3bea6f0a80b66ae8dd2da250b73" - integrity sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA== - ajv-keywords@^3.5.2: version "3.5.2" resolved "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz" @@ -1024,7 +1014,7 @@ browser-stdout@1.3.1: resolved "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz" integrity sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw== -browserslist@^4.14.5: +browserslist@^4.14.5, browserslist@^4.21.3, browserslist@^4.21.4: version "4.21.5" resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.5.tgz#75c5dae60063ee641f977e00edd3cfb2fb7af6a7" integrity sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w== @@ -1034,16 +1024,6 @@ browserslist@^4.14.5: node-releases "^2.0.8" update-browserslist-db "^1.0.10" -browserslist@^4.21.3, browserslist@^4.21.4: - version "4.21.4" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.4.tgz#e7496bbc67b9e39dd0f98565feccdcb0d4ff6987" - integrity sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw== - dependencies: - caniuse-lite "^1.0.30001400" - electron-to-chromium "^1.4.251" - node-releases "^2.0.6" - update-browserslist-db "^1.0.9" - buffer-from@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5" @@ -1085,11 +1065,6 @@ camelcase@^6.0.0: resolved "https://registry.npmjs.org/camelcase/-/camelcase-6.2.0.tgz" integrity sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg== -caniuse-lite@^1.0.30001400: - version "1.0.30001439" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001439.tgz#ab7371faeb4adff4b74dad1718a6fd122e45d9cb" - integrity sha512-1MgUzEkoMO6gKfXflStpYgZDlFM7M/ck/bgfVCACO5vnAf0fXoNVHdWtqGU+MYca+4bL9Z5bpOVmR33cWW9G2A== - caniuse-lite@^1.0.30001449: version "1.0.30001457" resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001457.tgz#6af34bb5d720074e2099432aa522c21555a18301" @@ -1415,11 +1390,6 @@ ee-first@1.1.1: resolved "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= -electron-to-chromium@^1.4.251: - version "1.4.284" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.284.tgz#61046d1e4cab3a25238f6bf7413795270f125592" - integrity sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA== - electron-to-chromium@^1.4.284: version "1.4.302" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.302.tgz#5770646ffe7051677b489226144aad9386d420f2" @@ -2666,30 +2636,18 @@ micromatch@^4.0.4: braces "^3.0.2" picomatch "^2.3.1" -mime-db@1.44.0: - version "1.44.0" - resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz" - integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg== - mime-db@1.52.0: version "1.52.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.1.27, mime-types@~2.1.34: +mime-types@^2.1.27, mime-types@~2.1.24, mime-types@~2.1.34: version "2.1.35" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== dependencies: mime-db "1.52.0" -mime-types@~2.1.24: - version "2.1.27" - resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz" - integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w== - dependencies: - mime-db "1.44.0" - mime@1.6.0: version "1.6.0" resolved "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz" @@ -2806,11 +2764,6 @@ neo-async@^2.6.2: resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== -node-releases@^2.0.6: - version "2.0.8" - resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.8.tgz#0f349cdc8fcfa39a92ac0be9bc48b7706292b9ae" - integrity sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A== - node-releases@^2.0.8: version "2.0.10" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.10.tgz#c311ebae3b6a148c89b1813fd7c4d3c024ef537f" @@ -3450,16 +3403,7 @@ scheduler@^0.19.1: loose-envify "^1.1.0" object-assign "^4.1.1" -schema-utils@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-3.0.0.tgz" - integrity sha512-6D82/xSzO094ajanoOSbe4YvXWMfn2A//8Y1+MUqFAJul5Bs+yn36xbK9OtNDcRVSBJ9jjeoXftM6CfztsjOAA== - dependencies: - "@types/json-schema" "^7.0.6" - ajv "^6.12.5" - ajv-keywords "^3.5.2" - -schema-utils@^3.1.0, schema-utils@^3.1.1: +schema-utils@^3.0.0, schema-utils@^3.1.0, schema-utils@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-3.1.1.tgz#bc74c4b6b6995c1d88f76a8b77bea7219e0c8281" integrity sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw== @@ -3885,7 +3829,7 @@ unpipe@1.0.0, unpipe@~1.0.0: resolved "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= -update-browserslist-db@^1.0.10, update-browserslist-db@^1.0.9: +update-browserslist-db@^1.0.10: version "1.0.10" resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz#0f54b876545726f17d00cd9a2561e6dade943ff3" integrity sha512-OztqDenkfFkbSG+tRxBeAnCVPckDBcvibKd35yDONx6OU8N7sqgwc7rCbkJ/WcYtVRZ4ba68d6byhC21GFh7sQ== @@ -3965,9 +3909,9 @@ webpack-sources@^3.2.3: integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== webpack@^5.75.0: - version "5.75.0" - resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.75.0.tgz#1e440468647b2505860e94c9ff3e44d5b582c152" - integrity sha512-piaIaoVJlqMsPtX/+3KTTO6jfvrSYgauFVdt8cr9LTHKmcq/AMd4mhzsiP7ZF/PGRNPGA8336jldh9l2Kt2ogQ== + version "5.76.0" + resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.76.0.tgz#f9fb9fb8c4a7dbdcd0d56a98e56b8a942ee2692c" + integrity sha512-l5sOdYBDunyf72HW8dF23rFtWq/7Zgvt/9ftMof71E/yUb1YLOBmTgA2K4vQthB3kotMrSj609txVE0dnr2fjA== dependencies: "@types/eslint-scope" "^3.7.3" "@types/estree" "^0.0.51" From 6d7d0a93267ced901cf64f41b75441535f533b7a Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Wed, 22 Mar 2023 10:37:47 +0200 Subject: [PATCH 36/45] Remove headers logging --- .../openvsx/web/LongRunningRequestFilter.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java b/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java index a73f00460..e5e2781fb 100644 --- a/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java +++ b/server/src/main/java/org/eclipse/openvsx/web/LongRunningRequestFilter.java @@ -21,7 +21,6 @@ import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; public class LongRunningRequestFilter extends OncePerRequestFilter { @@ -82,22 +81,6 @@ private void logWarning(HttpServletRequest request, HttpServletResponse response } } - builder.append("Headers:"); - var headerNames = request.getHeaderNames().asIterator(); - while(headerNames.hasNext()) { - var headerName = headerNames.next(); - var headerValues = new ArrayList(); - var headers = request.getHeaders(headerName).asIterator(); - while(headers.hasNext()) { - headerValues.add(headers.next()); - } - - builder.append("\n\t\t") - .append(headerName) - .append(": ") - .append(String.join(", ", headerValues)); - } - builder.append("\n\tResponse: ") .append(response.getStatus()); From 0a6dc3c4e09de6a2b39b28193f8e4fc82f5cc7b5 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Wed, 22 Mar 2023 11:17:02 +0200 Subject: [PATCH 37/45] Bump webui to 0.9.5 --- webui/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/package.json b/webui/package.json index 2a1b83cb7..7ab4cd713 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "openvsx-webui", - "version": "0.9.3", + "version": "0.9.5", "description": "User interface for Eclipse Open VSX", "keywords": [ "react", From 61e1ec659bce409bd8bc5d4ba8b287f5f6784082 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Wed, 22 Mar 2023 11:08:59 +0200 Subject: [PATCH 38/45] Make user managed entity --- .../src/main/java/org/eclipse/openvsx/UserService.java | 7 ++++++- .../src/test/java/org/eclipse/openvsx/UserAPITest.java | 9 +++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/UserService.java b/server/src/main/java/org/eclipse/openvsx/UserService.java index 7c3a0136a..e23b38269 100644 --- a/server/src/main/java/org/eclipse/openvsx/UserService.java +++ b/server/src/main/java/org/eclipse/openvsx/UserService.java @@ -306,7 +306,12 @@ public AccessTokenJson createAccessToken(UserData user, String description) { @Transactional public ResultJson deleteAccessToken(UserData user, long id) { var token = repositories.findAccessToken(id); - if (token == null || !token.isActive() || !token.getUser().equals(user)) { + if (token == null || !token.isActive()) { + throw new NotFoundException(); + } + + user = entityManager.merge(user); + if(!token.getUser().equals(user)) { throw new NotFoundException(); } diff --git a/server/src/test/java/org/eclipse/openvsx/UserAPITest.java b/server/src/test/java/org/eclipse/openvsx/UserAPITest.java index 9dbb358ad..563b74023 100644 --- a/server/src/test/java/org/eclipse/openvsx/UserAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/UserAPITest.java @@ -63,14 +63,17 @@ @WebMvcTest(UserAPI.class) @AutoConfigureWebClient @MockBean({ - EntityManager.class, EclipseService.class, ClientRegistrationRepository.class, StorageUtilService.class, - CacheService.class, ExtensionValidator.class, SimpleMeterRegistry.class + EclipseService.class, ClientRegistrationRepository.class, StorageUtilService.class, CacheService.class, + ExtensionValidator.class, SimpleMeterRegistry.class }) public class UserAPITest { @SpyBean UserService users; + @MockBean + EntityManager entityManager; + @MockBean RepositoryService repositories; @@ -153,6 +156,8 @@ public void testDeleteAccessToken() throws Exception { token.setActive(true); Mockito.when(repositories.findAccessToken(100)) .thenReturn(token); + Mockito.when(entityManager.merge(userData)) + .thenReturn(userData); mockMvc.perform(post("/user/token/delete/{id}", 100) .with(user("test_user")) From 925711ad002fce2c43e36ab4ad99c65fa858692e Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Wed, 22 Mar 2023 12:23:26 +0200 Subject: [PATCH 39/45] Fix 'Statement inside of curly braces should be on next line' --- webui/src/pages/admin-dashboard/namespace-change-dialog.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx b/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx index 7463b14f4..1d5a48cf3 100644 --- a/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx +++ b/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx @@ -52,6 +52,10 @@ const onClose = () => { props.onClose(); }; + const onInfoDialogClose = () => { + onClose(); + setInfoDialogIsOpen(false); + }; const onRemoveOldNamespaceChange = (event: React.ChangeEvent, checked: boolean) => { setRemoveOldNamespace(checked); }; @@ -120,6 +124,6 @@ - {props.onClose(); setInfoDialogIsOpen(false);}}/> + ; }; \ No newline at end of file From 29babd5cd601a10c4c82216858ff6c5e215f7488 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Wed, 22 Mar 2023 12:58:46 +0200 Subject: [PATCH 40/45] remove trailing space --- webui/src/pages/admin-dashboard/namespace-change-dialog.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx b/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx index 1d5a48cf3..0b1c746ba 100644 --- a/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx +++ b/webui/src/pages/admin-dashboard/namespace-change-dialog.tsx @@ -53,7 +53,7 @@ props.onClose(); }; const onInfoDialogClose = () => { - onClose(); + onClose(); setInfoDialogIsOpen(false); }; const onRemoveOldNamespaceChange = (event: React.ChangeEvent, checked: boolean) => { From 7c2b79e2799822daf7c0c0f02a7ea0011301d4bf Mon Sep 17 00:00:00 2001 From: yiningwang11 Date: Tue, 7 Feb 2023 14:30:50 -0500 Subject: [PATCH 41/45] add set up guide for MacOS --- doc/development.md | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/doc/development.md b/doc/development.md index 249a6c179..13263d57b 100644 --- a/doc/development.md +++ b/doc/development.md @@ -49,11 +49,51 @@ To get started quickly, it is recommended to use Gitpod as default with the deve - [https://learn.microsoft.com/en-us/windows/wsl/tutorials/wsl-containers](https://learn.microsoft.com/en-us/windows/wsl/tutorials/wsl-containers) - [https://www.youtube.com/watch?v=idW-an99TAM](https://www.youtube.com/watch?v=idW-an99TAM) -- Instal Elasticsearch with Docker +- Install Elasticsearch with Docker - sudo docker pull elasticsearch:7.9.3 - sudo docker run -d -name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.9.3 +### Setup locally on MacOS + +- Set up postgresql using homebrew + + - brew install postgresql@12 + + - Use "postgres –V" to check if it is the correct version + + - brew services start postgresql@12 + + - From the terminal, run the command "createdb postgres" (NOT inside psql) + + - "psql –d postgres" to enter the database + + - CREATE ROLE gitpod with LOGIN PASSWORD 'gitpod'; + + - Using \d to display all tables and relations, now empty + +- Make sure using the correct java version + + -Download java 11 if not yet downloaded yet + + - Run the command "/usr/libexec/java_home –V" to see your matching java virtual machines + + - Pick java 11 accordingly + + - export JAVA_HOME='/usr/libexec/java_home –v 11.0.18' + + - Run "java –version" to check if it's now running java 11 + + - https://stackoverflow.com/questions/21964709/how-to-set-or-change-the-default-java-jdk-version-on-macos + +- Set up elasticsearch using docker + +- Download docker desktop from https://www.docker.com/ + +- Download the elasticsearch image from docker hub and enable it in the docker desktop + +### Run the application + - cd cli - yarn install From fbc1d8e9b698d605d1fe97a3c26d40b25a2bfe6e Mon Sep 17 00:00:00 2001 From: yiningwang11 Date: Thu, 23 Feb 2023 09:56:13 -0500 Subject: [PATCH 42/45] updated setup details --- doc/development.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/development.md b/doc/development.md index 13263d57b..29e087ca1 100644 --- a/doc/development.md +++ b/doc/development.md @@ -56,41 +56,41 @@ To get started quickly, it is recommended to use Gitpod as default with the deve ### Setup locally on MacOS -- Set up postgresql using homebrew +- Set up postgreSQL using Homebrew - brew install postgresql@12 - - Use "postgres –V" to check if it is the correct version + - Use `postgres –V` to check if it is the correct version - brew services start postgresql@12 - - From the terminal, run the command "createdb postgres" (NOT inside psql) + - From the terminal, run the command `createdb postgres` (NOT inside `psql`) - - "psql –d postgres" to enter the database + - `psql –d postgres` to enter the database - CREATE ROLE gitpod with LOGIN PASSWORD 'gitpod'; - Using \d to display all tables and relations, now empty -- Make sure using the correct java version +- Make sure the correct Java version is being used - -Download java 11 if not yet downloaded yet + -Download Java 11 if you don't have it already - - Run the command "/usr/libexec/java_home –V" to see your matching java virtual machines + - Run the command `/usr/libexec/java_home –V` to see your matching Java virtual machines - - Pick java 11 accordingly + - Pick Java 11 accordingly - export JAVA_HOME='/usr/libexec/java_home –v 11.0.18' - - Run "java –version" to check if it's now running java 11 + - Run `java –version` to check if Java 11 is indeed being used - https://stackoverflow.com/questions/21964709/how-to-set-or-change-the-default-java-jdk-version-on-macos -- Set up elasticsearch using docker +- Set up Elasticsearch using Docker -- Download docker desktop from https://www.docker.com/ +- Download Docker Desktop from https://www.docker.com/ -- Download the elasticsearch image from docker hub and enable it in the docker desktop +- Download the Elasticsearch image from Docker Hub and enable it in the Docker Desktop ### Run the application From 07db44bd63daaee26018ef49d711e7cbe6d86a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Tron=C3=AD=C4=8Dek?= Date: Mon, 27 Mar 2023 17:49:45 +0200 Subject: [PATCH 43/45] Small fixes --- doc/development.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/development.md b/doc/development.md index 29e087ca1..2127d8efe 100644 --- a/doc/development.md +++ b/doc/development.md @@ -54,7 +54,7 @@ To get started quickly, it is recommended to use Gitpod as default with the deve - sudo docker pull elasticsearch:7.9.3 - sudo docker run -d -name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:7.9.3 -### Setup locally on MacOS +### Setup locally on macOS - Set up postgreSQL using Homebrew @@ -74,7 +74,7 @@ To get started quickly, it is recommended to use Gitpod as default with the deve - Make sure the correct Java version is being used - -Download Java 11 if you don't have it already + - Download Java 11 if you don't have it already - Run the command `/usr/libexec/java_home –V` to see your matching Java virtual machines From ec78245d31509fe418f1075faa71e08013091512 Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Thu, 23 Mar 2023 10:49:30 +0200 Subject: [PATCH 44/45] Need to deal with collisions when renaming and merging namespaces Merge namespace members Return error if namespaces contain duplicate extensions --- .../org/eclipse/openvsx/AdminService.java | 26 +++++++++++++++++++ .../openvsx/ChangeNamespaceService.java | 19 ++++++++++---- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/eclipse/openvsx/AdminService.java b/server/src/main/java/org/eclipse/openvsx/AdminService.java index 13337b584..53eb6c864 100644 --- a/server/src/main/java/org/eclipse/openvsx/AdminService.java +++ b/server/src/main/java/org/eclipse/openvsx/AdminService.java @@ -31,6 +31,7 @@ import java.time.DateTimeException; import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Comparator; import java.util.LinkedHashSet; import java.util.stream.Collectors; @@ -218,6 +219,31 @@ public void changeNamespace(ChangeNamespaceJson json) { if (newNamespace != null && !json.mergeIfNewNamespaceAlreadyExists) { throw new ErrorResultException("New namespace already exists: " + json.newNamespace); } + if (newNamespace != null) { + var newExtensions = repositories.findExtensions(newNamespace).stream() + .collect(Collectors.toMap(Extension::getName, e -> e)); + var oldExtensions = repositories.findExtensions(oldNamespace).stream() + .collect(Collectors.toMap(Extension::getName, e -> e)); + + var duplicateExtensions = oldExtensions.keySet().stream() + .filter(newExtensions::containsKey) + .collect(Collectors.joining("','")); + if(!duplicateExtensions.isEmpty()) { + var message = "Can't merge namespaces, because new namespace '" + + json.newNamespace + + "' and old namespace '" + + json.oldNamespace + + "' have " + + (duplicateExtensions.indexOf(',') == -1 ? "a " : "") + + "duplicate extension" + + (duplicateExtensions.indexOf(',') == -1 ? "" : "s") + + ": '" + + duplicateExtensions + + "'."; + + throw new ErrorResultException(message); + } + } scheduler.enqueue(new ChangeNamespaceJobRequest(json)); } diff --git a/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceService.java b/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceService.java index d4c44c876..a8b0065b9 100644 --- a/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceService.java +++ b/server/src/main/java/org/eclipse/openvsx/ChangeNamespaceService.java @@ -22,6 +22,7 @@ import javax.persistence.EntityManager; import javax.transaction.Transactional; import java.util.List; +import java.util.stream.Collectors; @Component public class ChangeNamespaceService { @@ -59,7 +60,7 @@ public void changeNamespaceInDatabase( } changeExtensionNamespace(extensions, newNamespace); - changeMembershipNamespace(oldNamespace, newNamespace); + changeMembershipNamespace(oldNamespace, newNamespace, removeOldNamespace); updatedResources.forEach(entityManager::merge); if(removeOldNamespace) { @@ -77,10 +78,18 @@ private void changeExtensionNamespace(Streamable extensions, Namespac } } - private void changeMembershipNamespace(Namespace oldNamespace, Namespace newNamespace) { - var memberships = repositories.findMemberships(oldNamespace); - for(var membership : memberships) { - membership.setNamespace(newNamespace); + private void changeMembershipNamespace(Namespace oldNamespace, Namespace newNamespace, boolean removeOldNamespace) { + var oldMemberships = repositories.findMemberships(oldNamespace).stream() + .collect(Collectors.toMap(m -> m.getUser().getId(), m -> m)); + var newMemberships = repositories.findMemberships(newNamespace).stream() + .collect(Collectors.toMap(m -> m.getUser().getId(), m -> m)); + + for(var entry : oldMemberships.entrySet()) { + if(!newMemberships.containsKey(entry.getKey())) { + entry.getValue().setNamespace(newNamespace); + } else if (removeOldNamespace) { + entityManager.remove(entry.getValue()); + } } } } From 082d053f3635ec10f4d05a2f6a9fdfad003342ac Mon Sep 17 00:00:00 2001 From: amvanbaren Date: Fri, 31 Mar 2023 14:46:17 +0300 Subject: [PATCH 45/45] Merge ExtensionVersion before persisting FileResource --- .../eclipse/openvsx/publish/PublishExtensionVersionService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java index d44ad3a60..9576e74fb 100644 --- a/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java +++ b/server/src/main/java/org/eclipse/openvsx/publish/PublishExtensionVersionService.java @@ -80,6 +80,7 @@ public void mirrorResource(FileResource resource) { @Transactional public void persistResource(FileResource resource) { + resource.setExtension(entityManager.merge(resource.getExtension())); entityManager.persist(resource); }