diff --git a/docker-compose.yml b/docker-compose.yml index d1a892f73..50831e44b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,157 @@ services: profiles: - kibana + redis-node-1: + image: redis:7.2 + container_name: redis-node-1 + ports: + - "7001:7000" + command: redis-server /etc/redis.conf + volumes: + - ./redis.conf:/etc/redis.conf + - /redis/node-1:/data + environment: + - REDISCLI_AUTH=openvsx + healthcheck: + test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"] + interval: 5s + retries: 5 + profiles: + - redis + + redis-node-2: + image: redis:7.2 + container_name: redis-node-2 + depends_on: + redis-node-1: + condition: service_healthy + ports: + - "7002:7000" + command: redis-server /etc/redis.conf + volumes: + - ./redis.conf:/etc/redis.conf + - /redis/node-2:/data + environment: + - REDISCLI_AUTH=openvsx + healthcheck: + test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"] + interval: 5s + retries: 5 + profiles: + - redis + + redis-node-3: + image: redis:7.2 + container_name: redis-node-3 + depends_on: + redis-node-2: + condition: service_healthy + ports: + - "7003:7000" + command: redis-server /etc/redis.conf + volumes: + - ./redis.conf:/etc/redis.conf + - /redis/node-3:/data + environment: + - REDISCLI_AUTH=openvsx + healthcheck: + test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"] + interval: 5s + retries: 5 + profiles: + - redis + + redis-node-4: + image: redis:7.2 + container_name: redis-node-4 + depends_on: + redis-node-3: + condition: service_healthy + ports: + - "7004:7000" + command: redis-server /etc/redis.conf + volumes: + - ./redis.conf:/etc/redis.conf + - /redis/node-4:/data + environment: + - REDISCLI_AUTH=openvsx + healthcheck: + test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"] + interval: 5s + retries: 5 + profiles: + - redis + + redis-node-5: + image: redis:7.2 + container_name: redis-node-5 + depends_on: + redis-node-4: + condition: service_healthy + ports: + - "7005:7000" + command: redis-server /etc/redis.conf + volumes: + - ./redis.conf:/etc/redis.conf + - /redis/node-5:/data + environment: + - REDISCLI_AUTH=openvsx + healthcheck: + test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"] + interval: 5s + retries: 5 + profiles: + - redis + + redis-node-6: + image: redis:7.2 + container_name: redis-node-6 + depends_on: + redis-node-5: + condition: service_healthy + ports: + - "7006:7000" + command: redis-server /etc/redis.conf + volumes: + - ./redis.conf:/etc/redis.conf + - /redis/node-6:/data + environment: + - REDISCLI_AUTH=openvsx + healthcheck: + test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"] + interval: 5s + retries: 5 + profiles: + - redis + + redis-cluster-init: + image: redis:7.2 + depends_on: + - redis-node-1 + - redis-node-2 + - redis-node-3 + - redis-node-4 + - redis-node-5 + - redis-node-6 + environment: + - REDISCLI_AUTH=openvsx + entrypoint: > + bash -c " + sleep 10; + echo yes | redis-cli --user openvsx --cluster create + redis-node-1:7000 redis-node-2:7000 redis-node-3:7000 + redis-node-4:7000 redis-node-5:7000 redis-node-6:7000 + --cluster-replicas 1" + profiles: + - redis + + redisinsight: + image: redis/redisinsight + ports: + - '5540:5540' + profiles: + - redisinsight + server: image: openjdk:17 working_dir: /app diff --git a/redis.conf b/redis.conf new file mode 100644 index 000000000..8a0381af1 --- /dev/null +++ b/redis.conf @@ -0,0 +1,12 @@ +# Configuration for Redis nodes in docker-compose.yml +port 7000 +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 5000 +appendonly yes +maxmemory 64mb +maxmemory-policy allkeys-lru +user default off +user openvsx on >openvsx ~* +@all +masteruser openvsx +masterauth openvsx \ No newline at end of file diff --git a/server/build.gradle b/server/build.gradle index cbf513b95..e1ca06c43 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -34,14 +34,15 @@ def versions = [ woodstox: '6.4.0', jobrunr: '7.5.0', bucket4j: '0.12.7', - ehcache: '3.10.8', + bucket4j_redis: '8.10.1', tika: '3.1.0', bouncycastle: '1.80', commons_lang3: '3.12.0', jaxb_api: '2.3.1', jaxb_impl: '2.3.8', - gatling: '3.13.5', - loki4j: '1.4.2' + gatling: '3.9.5', + loki4j: '1.4.2', + embedded_redis: '1.4.3' ] ext['junit-jupiter.version'] = versions.junit sourceCompatibility = versions.java @@ -80,6 +81,7 @@ dependencies { implementation "org.springframework.boot:spring-boot-starter-jooq" implementation "org.springframework.boot:spring-boot-starter-data-jpa" implementation "org.springframework.boot:spring-boot-starter-data-elasticsearch" + implementation "org.springframework.boot:spring-boot-starter-data-redis" implementation "org.springframework.boot:spring-boot-starter-security" implementation "org.springframework.boot:spring-boot-starter-actuator" implementation "org.springframework.boot:spring-boot-starter-cache" @@ -89,8 +91,9 @@ dependencies { implementation "org.springframework.session:spring-session-jdbc" implementation "org.springframework.retry:spring-retry" implementation "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}" - implementation "org.ehcache:ehcache:${versions.ehcache}" + implementation "com.github.ben-manes.caffeine:caffeine" implementation "com.giffing.bucket4j.spring.boot.starter:bucket4j-spring-boot-starter:${versions.bucket4j}" + implementation "com.bucket4j:bucket4j-redis:${versions.bucket4j_redis}" implementation "org.jobrunr:jobrunr-spring-boot-3-starter:${versions.jobrunr}" implementation "org.flywaydb:flyway-core:${versions.flyway}" implementation "com.google.cloud:google-cloud-storage:${versions.gcloud}" @@ -112,6 +115,7 @@ dependencies { implementation "io.micrometer:micrometer-tracing" implementation "io.micrometer:micrometer-tracing-bridge-otel" implementation "io.opentelemetry:opentelemetry-exporter-zipkin" + implementation "com.github.codemonstur:embedded-redis:${versions.embedded_redis}" runtimeOnly "io.micrometer:micrometer-registry-prometheus" runtimeOnly "org.postgresql:postgresql" jooqGenerator "org.postgresql:postgresql" diff --git a/server/src/dev/resources/application.yml b/server/src/dev/resources/application.yml index 5a949c00f..4b37d7ece 100644 --- a/server/src/dev/resources/application.yml +++ b/server/src/dev/resources/application.yml @@ -16,9 +16,16 @@ spring: exclude: org.springframework.boot.actuate.autoconfigure.tracing.zipkin.ZipkinAutoConfiguration profiles: include: ovsx - cache: - jcache: - config: classpath:ehcache.xml + data: + redis: + # redis standalone configuration + host: localhost + port: 6379 + # connect to redis cluster configured in docker-compose.yml +# cluster: +# nodes: '127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003,127.0.0.1:7004,127.0.0.1:7005,127.0.0.1:7006' +# username: openvsx +# password: openvsx datasource: url: jdbc:postgresql://localhost:5432/postgres username: gitpod @@ -101,6 +108,7 @@ org: bucket4j: enabled: true + cache-to-use: redis-jedis # use redis-cluster-jedis when running redis cluster filters: - cache-name: buckets url: '/api/-/(namespace/create|publish)' @@ -140,7 +148,10 @@ ovsx: databasesearch: enabled: false elasticsearch: + enabled: true clear-on-start: true + redis: + embedded: true eclipse: base-url: https://api.eclipse.org publisher-agreement: diff --git a/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala b/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala index 27da29c8f..71b6a6f22 100644 --- a/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala +++ b/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala @@ -318,7 +318,7 @@ object Scenarios { val categories = Array("", "Programming Languages", "Themes", "Snippets", "Debuggers", "Linters", "Other") val sizes = Array("", "5", "500") val offsets = Array("", "1", "100", "25000") - val sortBys = Array("", "relevance", "timestamp", "averageRating", "downloadCount") + val sortBys = Array("", "relevance", "timestamp", "rating", "downloadCount") val sortOrders = Array("", "asc", "desc") val includeAllVersions = Array("", "true", "false") diff --git a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java index 3d20ecff6..85eecef5b 100644 --- a/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java +++ b/server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java @@ -21,6 +21,7 @@ import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.ExtensionSearch; import org.eclipse.openvsx.search.ISearchService; +import org.eclipse.openvsx.search.SearchResult; import org.eclipse.openvsx.search.SearchUtilService; import org.eclipse.openvsx.storage.StorageUtilService; import org.eclipse.openvsx.util.*; @@ -29,7 +30,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.data.domain.PageRequest; -import org.springframework.data.elasticsearch.core.SearchHits; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; @@ -257,11 +257,11 @@ public SearchResultJson search(ISearchService.Options options) { return json; } - var searchHits = search.search(options); - if(searchHits.hasSearchHits()) { - json.setExtensions(toSearchEntries(searchHits, options)); + var result = search.search(options); + if(result.hasSearchHits()) { + json.setExtensions(toSearchEntries(result, options)); json.setOffset(options.requestedOffset()); - json.setTotalSize((int) searchHits.getTotalHits()); + json.setTotalSize((int) result.getTotalHits()); } else { json.setExtensions(Collections.emptyList()); } @@ -724,9 +724,9 @@ public ResultJson deleteReview(String namespace, String extensionName) { return ResultJson.success("Deleted review for " + NamingUtil.toExtensionId(extension)); } - private LinkedHashMap getLatestVersions(SearchHits searchHits) { - var ids = searchHits.stream() - .map(searchHit -> searchHit.getContent().getId()) + private LinkedHashMap getLatestVersions(SearchResult result) { + var ids = result.getHits().stream() + .map(ExtensionSearch::getId) .distinct() .collect(Collectors.toList()); @@ -756,9 +756,9 @@ private LinkedHashMap findLatestVersions(Collection toSearchEntries(SearchHits searchHits, ISearchService.Options options) { + private List toSearchEntries(SearchResult result, ISearchService.Options options) { var serverUrl = UrlUtil.getBaseUrl(); - var latestVersions = getLatestVersions(searchHits); + var latestVersions = getLatestVersions(result); var membershipsByNamespaceId = getMemberships(latestVersions.values()); var searchEntries = latestVersions.entrySet().stream() .map(e -> { diff --git a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java index ad0776e5e..fd821b127 100644 --- a/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java +++ b/server/src/main/java/org/eclipse/openvsx/RegistryAPI.java @@ -22,6 +22,7 @@ import org.eclipse.openvsx.entities.SemanticVersion; import org.eclipse.openvsx.json.*; import org.eclipse.openvsx.search.ISearchService; +import org.eclipse.openvsx.search.SortBy; import org.eclipse.openvsx.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -757,8 +758,8 @@ public ResponseEntity search( @RequestParam(defaultValue = "desc") @Parameter(description = "Descending or ascending sort order", schema = @Schema(type = "string", allowableValues = {"asc", "desc"})) String sortOrder, - @RequestParam(defaultValue = "relevance") - @Parameter(description = "Sort key (relevance is a weighted mix of various properties)", schema = @Schema(type = "string", allowableValues = {"relevance", "timestamp", "averageRating", "downloadCount"})) + @RequestParam(defaultValue = SortBy.RELEVANCE) + @Parameter(description = "Sort key (relevance is a weighted mix of various properties)", schema = @Schema(type = "string", allowableValues = {SortBy.RELEVANCE, SortBy.TIMESTAMP, SortBy.RATING, SortBy.DOWNLOADS})) String sortBy, @RequestParam(defaultValue = "false") @Parameter(description = "Whether to include information on all available versions for each returned entry") diff --git a/server/src/main/java/org/eclipse/openvsx/RegistryApplication.java b/server/src/main/java/org/eclipse/openvsx/RegistryApplication.java index 89c24ad53..523660e9d 100644 --- a/server/src/main/java/org/eclipse/openvsx/RegistryApplication.java +++ b/server/src/main/java/org/eclipse/openvsx/RegistryApplication.java @@ -35,7 +35,6 @@ @EnableScheduling @EnableRetry @EnableAsync -@EnableCaching(proxyTargetClass = true) @EnableConfigurationProperties(OAuth2AttributesConfig.class) public class RegistryApplication { diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java b/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java index 0ca11e508..7f196bc86 100644 --- a/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java +++ b/server/src/main/java/org/eclipse/openvsx/adapter/LocalVSCodeService.java @@ -18,7 +18,9 @@ import org.eclipse.openvsx.entities.FileResource; import org.eclipse.openvsx.publish.ExtensionVersionIntegrityService; import org.eclipse.openvsx.repositories.RepositoryService; +import org.eclipse.openvsx.search.ExtensionSearch; import org.eclipse.openvsx.search.SearchUtilService; +import org.eclipse.openvsx.search.SortBy; import org.eclipse.openvsx.storage.StorageUtilService; import org.eclipse.openvsx.util.*; import org.slf4j.Logger; @@ -92,7 +94,7 @@ public ExtensionQueryResult extensionQuery(ExtensionQueryParam param, int defaul if (param.filters() == null || param.filters().isEmpty()) { pageNumber = 0; pageSize = defaultPageSize; - sortBy = "relevance"; + sortBy = SortBy.RELEVANCE; sortOrder = "desc"; targetPlatform = null; extensionIds = Collections.emptySet(); @@ -138,8 +140,8 @@ public ExtensionQueryResult extensionQuery(ExtensionQueryParam param, int defaul var searchResult = search.search(searchOptions); totalCount = searchResult.getTotalHits(); - var ids = searchResult.getSearchHits().stream() - .map(hit -> hit.getContent().getId()) + var ids = searchResult.getHits().stream() + .map(ExtensionSearch::getId) .collect(Collectors.toList()); var extensionsMap = repositories.findActiveExtensionsById(ids).stream() @@ -255,13 +257,13 @@ public ExtensionQueryResult toQueryResult(List e private String getSortBy(int sortBy) { switch (sortBy) { case 4: // InstallCount - return "downloadCount"; + return SortBy.DOWNLOADS; case 5: // PublishedDate - return "timestamp"; + return SortBy.TIMESTAMP; case 6: // AverageRating - return "averageRating"; + return SortBy.RATING; default: - return "relevance"; + return SortBy.RELEVANCE; } } @@ -325,22 +327,22 @@ public ResponseEntity getAsset( return storageUtil.getFileResponse(resource); } else if(asset.startsWith(FILE_WEB_RESOURCES + "/extension/")) { var name = asset.substring((FILE_WEB_RESOURCES.length() + 1)); - var file = getWebResource(namespace, extensionName, targetPlatform, version, name, false); - return storageUtil.getFileResponse(file); + var extensionDownloadPath = webResources.getExtensionDownload(namespace, extensionName, targetPlatform, version); + var file = extensionDownloadPath != null ? getWebResource(namespace, extensionName, targetPlatform, version, name, extensionDownloadPath) : null; + if(file != null) { + return storageUtil.getFileResponse(file); + } } throw new NotFoundException(); } - private Path getWebResource(String namespaceName, String extensionName, String targetPlatform, String version, String name, boolean browse) { - var file = webResources.getWebResource(namespaceName, extensionName, targetPlatform, version, name, browse); - if(file == null) { - throw new NotFoundException(); - } - if(!Files.exists(file)) { + private Path getWebResource(String namespaceName, String extensionName, String targetPlatform, String version, String name, Path extensionDownloadPath) { + var file = webResources.getWebResource(namespaceName, extensionName, targetPlatform, version, name, extensionDownloadPath); + if(file != null && !Files.exists(file)) { logger.error("File doesn't exist {}", file); cache.evictWebResourceFile(namespaceName, extensionName, targetPlatform, version, name); - throw new NotFoundException(); + file = null; } return file; } @@ -401,8 +403,22 @@ public ResponseEntity browse(String namespaceName, String return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(builtinExtensionResponse()); } - var file = getWebResource(namespaceName, extensionName, null, version, path, true); - return storageUtil.getFileResponse(file); + var extensionDownloadPath = webResources.getExtensionDownload(namespaceName, extensionName, null, version); + if(extensionDownloadPath == null) { + throw new NotFoundException(); + } + + var file = getWebResource(namespaceName, extensionName, null, version, path, extensionDownloadPath); + if(file != null) { + return storageUtil.getFileResponse(file); + } + + var node = webResources.browseExtensionPackage(namespaceName, extensionName, null, version, path, extensionDownloadPath); + if(node != null) { + return storageUtil.getFileResponse(node); + } + + throw new NotFoundException(); } private ExtensionQueryResult.Extension toQueryExtension(Extension extension, ExtensionVersion latest, List versions, int flags) { diff --git a/server/src/main/java/org/eclipse/openvsx/adapter/WebResourceService.java b/server/src/main/java/org/eclipse/openvsx/adapter/WebResourceService.java index a17bd24b3..b2bbdc6f5 100644 --- a/server/src/main/java/org/eclipse/openvsx/adapter/WebResourceService.java +++ b/server/src/main/java/org/eclipse/openvsx/adapter/WebResourceService.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import io.micrometer.observation.annotation.Observed; import org.eclipse.openvsx.cache.CacheService; import org.eclipse.openvsx.cache.FilesCacheKeyGenerator; @@ -35,8 +36,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipFile; -import static org.eclipse.openvsx.cache.CacheService.CACHE_WEB_RESOURCE_FILES; -import static org.eclipse.openvsx.cache.CacheService.GENERATOR_FILES; +import static org.eclipse.openvsx.cache.CacheService.*; @Component public class WebResourceService { @@ -60,56 +60,62 @@ public WebResourceService( this.filesCacheKeyGenerator = filesCacheKeyGenerator; } - @Observed - @Cacheable(value = CACHE_WEB_RESOURCE_FILES, keyGenerator = GENERATOR_FILES) - public Path getWebResource(String namespace, String extension, String targetPlatform, String version, String name, boolean browse) { + public Path getExtensionDownload(String namespace, String extension, String targetPlatform, String version) { var download = repositories.findFileByType(namespace, extension, targetPlatform, version, FileResource.DOWNLOAD); if(download == null) { return null; } var path = storageUtil.getCachedFile(download); - if(path == null) { - return null; - } - if(!Files.exists(path)) { + if(path != null && !Files.exists(path)) { logger.error("File doesn't exist {}", path); cache.evictExtensionFile(download); - return null; + path = null; } - try(var zip = new ZipFile(path.toFile())) { + return path; + } + + @Observed + @Cacheable(value = CACHE_WEB_RESOURCE_FILES, keyGenerator = GENERATOR_FILES, cacheManager = "fileCacheManager") + public Path getWebResource(String namespace, String extension, String targetPlatform, String version, String name, Path extensionDownloadPath) { + try(var zip = new ZipFile(extensionDownloadPath.toFile())) { var fileEntry = zip.getEntry(name); if(fileEntry != null) { var fileExt = getFileExtension(fileEntry); var file = filesCacheKeyGenerator.generateCachedWebResourcePath(namespace, extension, targetPlatform, version, name, fileExt); writeBinaryFile(file, zip, fileEntry); return file; - } else if (browse) { - var dirName = getDirectoryName(name); - var dirEntries = zip.stream() - .filter(entry -> entry.getName().startsWith(dirName)) - .map(entry -> getFileInDirectory(dirName, entry)) - .collect(Collectors.toSet()); - if(dirEntries.isEmpty()) { - return null; - } - - var baseUrl = UrlUtil.createApiUrl(UrlUtil.getBaseUrl(), "vscode", "unpkg", namespace, extension, version); - var mapper = new ObjectMapper(); - var node = mapper.createArrayNode(); - for (var entry : dirEntries) { - node.add(baseUrl + "/" + entry); - } - - var file = filesCacheKeyGenerator.generateCachedWebResourcePath(namespace, extension, targetPlatform, version, name, ".unpkg.json"); - writeJsonFile(file, mapper, node); - return file; } else { return null; } } catch (IOException | UncheckedIOException e) { - throw new ErrorResultException("Failed to read extension files for " + NamingUtil.toLogFormat(download.getExtension()), HttpStatus.INTERNAL_SERVER_ERROR); + throw new ErrorResultException("Failed to read extension files for " + NamingUtil.toLogFormat(namespace, extension, targetPlatform, version), HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + @Cacheable(value = CACHE_BROWSE_EXTENSION_FILES, keyGenerator = GENERATOR_FILES, cacheManager = "fileCacheManager") + public ArrayNode browseExtensionPackage(String namespace, String extension, String targetPlatform, String version, String name, Path extensionDownloadPath) { + try(var zip = new ZipFile(extensionDownloadPath.toFile())) { + var dirName = getDirectoryName(name); + var dirEntries = zip.stream() + .filter(entry -> entry.getName().startsWith(dirName)) + .map(entry -> getFileInDirectory(dirName, entry)) + .collect(Collectors.toSet()); + if(dirEntries.isEmpty()) { + return null; + } + + var baseUrl = UrlUtil.createApiUrl("", "vscode", "unpkg", namespace, extension, version); + var mapper = new ObjectMapper(); + var node = mapper.createArrayNode(); + for (var entry : dirEntries) { + node.add(baseUrl + "/" + entry); + } + + return node; + } catch (IOException | UncheckedIOException e) { + throw new ErrorResultException("Failed to read extension files for " + NamingUtil.toLogFormat(namespace, extension, targetPlatform, version), HttpStatus.INTERNAL_SERVER_ERROR); } } diff --git a/server/src/main/java/org/eclipse/openvsx/cache/CacheConfig.java b/server/src/main/java/org/eclipse/openvsx/cache/CacheConfig.java new file mode 100644 index 000000000..4e0ac35a5 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/cache/CacheConfig.java @@ -0,0 +1,185 @@ +/** ****************************************************************************** + * Copyright (c) 2025 Precies. Software OU and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.cache; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.giffing.bucket4j.spring.boot.starter.context.properties.Bucket4JBootProperties; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Scheduler; +import io.micrometer.common.util.StringUtils; +import org.eclipse.openvsx.entities.ExtensionVersion; +import org.eclipse.openvsx.json.ExtensionJson; +import org.eclipse.openvsx.json.NamespaceDetailsJson; +import org.eclipse.openvsx.search.SearchResult; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.data.redis.RedisProperties; +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.cache.caffeine.CaffeineCacheManager; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.redis.cache.RedisCacheConfiguration; +import org.springframework.data.redis.cache.RedisCacheManager; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.serializer.*; +import redis.clients.jedis.DefaultJedisClientConfig; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisPool; + +import java.time.Duration; +import java.util.stream.Collectors; + +import static org.eclipse.openvsx.cache.CacheService.*; + +@Configuration +@EnableCaching(proxyTargetClass = true) +public class CacheConfig { + + @Bean + public Cache extensionCache( + @Value("${ovsx.caching.files-extension.tti:PT1H}") Duration timeToIdle, + @Value("${ovsx.caching.files-extension.max-size:20}") long maxSize + ) { + return Caffeine.newBuilder() + .removalListener(new ExpiredFileListener()) + .expireAfterAccess(timeToIdle) + .maximumSize(maxSize) + .scheduler(Scheduler.systemScheduler()) + .recordStats() + .build(); + } + + @Bean + public Cache webResourceCache( + @Value("${ovsx.caching.files-webresource.tti:PT1H}") Duration timeToIdle, + @Value("${ovsx.caching.files-webresource.max-size:150}") long maxSize + ) { + return Caffeine.newBuilder() + .removalListener(new ExpiredFileListener()) + .expireAfterAccess(timeToIdle) + .maximumSize(maxSize) + .scheduler(Scheduler.systemScheduler()) + .recordStats() + .build(); + } + + @Bean + public Cache browseCache( + @Value("${ovsx.caching.files-browse.tti:PT1H}") Duration timeToIdle, + @Value("${ovsx.caching.files-browse.max-size:50}") long maxSize + ) { + return Caffeine.newBuilder() + .expireAfterAccess(timeToIdle) + .maximumSize(maxSize) + .scheduler(Scheduler.systemScheduler()) + .recordStats() + .build(); + } + + @Bean + public CacheManager fileCacheManager( + Cache extensionCache, + Cache webResourceCache, + Cache browseCache + ) { + CaffeineCacheManager caffeineCacheManager = new CaffeineCacheManager(); + caffeineCacheManager.registerCustomCache(CACHE_EXTENSION_FILES, extensionCache); + caffeineCacheManager.registerCustomCache(CACHE_WEB_RESOURCE_FILES, webResourceCache); + caffeineCacheManager.registerCustomCache(CACHE_BROWSE_EXTENSION_FILES, browseCache); + return caffeineCacheManager; + } + + @Bean + @ConditionalOnProperty(prefix = Bucket4JBootProperties.PROPERTY_PREFIX, name = "cache-to-use", havingValue = "redis-jedis") + public JedisPool jedisPool(RedisProperties properties) { + return new JedisPool(properties.getHost(), properties.getPort(), properties.getUsername(), properties.getPassword()); + } + + @Bean + @ConditionalOnProperty(prefix = Bucket4JBootProperties.PROPERTY_PREFIX, name = "cache-to-use", havingValue = "redis-cluster-jedis") + public JedisCluster jedisCluster(RedisProperties properties) { + var configBuilder = DefaultJedisClientConfig.builder(); + var username = properties.getUsername(); + if(StringUtils.isNotEmpty(username)) { + configBuilder.user(username); + } + var password = properties.getPassword(); + if(StringUtils.isNotEmpty(password)) { + configBuilder.password(password); + } + + var nodes = properties.getCluster().getNodes().stream() + .map(HostAndPort::from) + .collect(Collectors.toSet()); + + return new JedisCluster(nodes, configBuilder.build()); + } + + @Bean + @Primary + public CacheManager redisCacheManager( + RedisConnectionFactory redisConnectionFactory, + @Value("${ovsx.caching.average-review-rating.ttl:P3D}") Duration averageReviewRatingTtl, + @Value("${ovsx.caching.namespace-details-json.ttl:PT1H}") Duration namespaceDetailsJsonTtl, + @Value("${ovsx.caching.database-search.ttl:PT1H}") Duration databaseSearchTtl, + @Value("${ovsx.caching.extension-json.ttl:PT1H}") Duration extensionJsonTtl, + @Value("${ovsx.caching.latest-extension-version.ttl:PT1H}") Duration latestExtensionVersionTtl, + @Value("${ovsx.caching.sitemap.ttl:PT1H}") Duration sitemapTtl, + @Value("${ovsx.caching.malicious-extensions.ttl:P3D}") Duration maliciousExtensionsTtl + ) { + var extensionVersionMapper = JsonMapper.builder() + .addModule(new JavaTimeModule()) + .serializationInclusion(JsonInclude.Include.NON_NULL) + .build(); + return RedisCacheManager.builder(redisConnectionFactory) + .withCacheConfiguration( + CACHE_AVERAGE_REVIEW_RATING, + redisCacheConfig(new GenericJackson2JsonRedisSerializer(), averageReviewRatingTtl) + ) + .withCacheConfiguration( + CACHE_NAMESPACE_DETAILS_JSON, + redisCacheConfig(new Jackson2JsonRedisSerializer<>(NamespaceDetailsJson.class), namespaceDetailsJsonTtl) + ) + .withCacheConfiguration( + CACHE_DATABASE_SEARCH, + redisCacheConfig(new Jackson2JsonRedisSerializer<>(SearchResult.class), databaseSearchTtl) + ) + .withCacheConfiguration( + CACHE_EXTENSION_JSON, + redisCacheConfig(new Jackson2JsonRedisSerializer<>(ExtensionJson.class), extensionJsonTtl) + ) + .withCacheConfiguration( + CACHE_LATEST_EXTENSION_VERSION, + redisCacheConfig(new Jackson2JsonRedisSerializer<>(extensionVersionMapper, ExtensionVersion.class), latestExtensionVersionTtl) + ) + .withCacheConfiguration( + CACHE_SITEMAP, + redisCacheConfig(new StringRedisSerializer(), sitemapTtl) + ) + .withCacheConfiguration( + CACHE_MALICIOUS_EXTENSIONS, + redisCacheConfig(new GenericJackson2JsonRedisSerializer(), maliciousExtensionsTtl) + ) + .build(); + } + + private RedisCacheConfiguration redisCacheConfig(RedisSerializer serializer, Duration ttl) { + var serializationPair = RedisSerializationContext.SerializationPair.fromSerializer(serializer); + return RedisCacheConfiguration.defaultCacheConfig() + .serializeValuesWith(serializationPair) + .entryTtl(ttl); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java index 9129af419..394d600f2 100644 --- a/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java +++ b/server/src/main/java/org/eclipse/openvsx/cache/CacheService.java @@ -25,6 +25,7 @@ public class CacheService { public static final String CACHE_DATABASE_SEARCH = "database.search"; public static final String CACHE_WEB_RESOURCE_FILES = "files.webresource"; + public static final String CACHE_BROWSE_EXTENSION_FILES = "files.browse"; public static final String CACHE_EXTENSION_FILES = "files.extension"; public static final String CACHE_EXTENSION_JSON = "extension.json"; public static final String CACHE_LATEST_EXTENSION_VERSION = "latest.extension.version"; diff --git a/server/src/main/java/org/eclipse/openvsx/cache/EmbeddedRedisServer.java b/server/src/main/java/org/eclipse/openvsx/cache/EmbeddedRedisServer.java new file mode 100644 index 000000000..e5e6e49c8 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/cache/EmbeddedRedisServer.java @@ -0,0 +1,42 @@ +/** ****************************************************************************** + * Copyright (c) 2025 Precies. Software OU and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.cache; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.autoconfigure.data.redis.RedisProperties; +import org.springframework.stereotype.Component; +import redis.embedded.RedisServer; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.IOException; + +@Component +@ConditionalOnProperty(value = "ovsx.redis.embedded", havingValue = "true") +public class EmbeddedRedisServer { + + private final int port; + private RedisServer server; + + public EmbeddedRedisServer(RedisProperties properties) { + port = properties.getPort(); + } + + @PostConstruct + public void start() throws IOException { + server = new RedisServer(port); + server.start(); + } + + @PreDestroy + public void stop() throws IOException { + server.stop(); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/cache/ExpiredFileListener.java b/server/src/main/java/org/eclipse/openvsx/cache/ExpiredFileListener.java index 4609361ea..fbbd89b83 100644 --- a/server/src/main/java/org/eclipse/openvsx/cache/ExpiredFileListener.java +++ b/server/src/main/java/org/eclipse/openvsx/cache/ExpiredFileListener.java @@ -9,9 +9,9 @@ * ****************************************************************************** */ package org.eclipse.openvsx.cache; -import org.ehcache.event.CacheEvent; -import org.ehcache.event.CacheEventListener; -import org.ehcache.event.EventType; +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.RemovalListener; +import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,15 +19,13 @@ import java.nio.file.Files; import java.nio.file.Path; -// V can be Path or NullValue (Spring's way of caching null values), that's why Object is used as V type -public class ExpiredFileListener implements CacheEventListener { +public class ExpiredFileListener implements RemovalListener { protected final Logger logger = LoggerFactory.getLogger(ExpiredFileListener.class); + @Override - public void onEvent(CacheEvent cacheEvent) { - logger.info("Expired file cache event: {} | key: {}", cacheEvent.getType(), cacheEvent.getKey()); - var oldValue = cacheEvent.getOldValue(); - var path = oldValue instanceof Path ? (Path) oldValue : null; - if(path == null || (cacheEvent.getType() == EventType.UPDATED && path.equals(cacheEvent.getNewValue()))) { + public void onRemoval(@Nullable Object key, @Nullable Object value, RemovalCause cause) { + logger.info("File removal cache event: {} | key: {} | value: {}", cause, key, value); + if(!(value instanceof Path path)) { return; } diff --git a/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterBucket4jConfiguration.java b/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterBucket4jConfiguration.java new file mode 100644 index 000000000..d9479b9e5 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterBucket4jConfiguration.java @@ -0,0 +1,61 @@ +/** ****************************************************************************** + * Copyright (c) 2025 Precies. Software OU and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.cache; + +import com.giffing.bucket4j.spring.boot.starter.config.cache.CacheManager; +import com.giffing.bucket4j.spring.boot.starter.config.cache.SyncCacheResolver; +import com.giffing.bucket4j.spring.boot.starter.config.condition.ConditionalOnCache; +import com.giffing.bucket4j.spring.boot.starter.config.condition.ConditionalOnFilterConfigCacheEnabled; +import com.giffing.bucket4j.spring.boot.starter.config.condition.ConditionalOnSynchronousPropertyCondition; +import com.giffing.bucket4j.spring.boot.starter.context.properties.Bucket4JBootProperties; +import com.giffing.bucket4j.spring.boot.starter.context.properties.Bucket4JConfiguration; +import io.github.bucket4j.redis.jedis.cas.JedisBasedProxyManager; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import redis.clients.jedis.JedisCluster; + +@Configuration +@ConditionalOnSynchronousPropertyCondition +@ConditionalOnClass(JedisBasedProxyManager.JedisBasedProxyManagerBuilder.class) +@ConditionalOnBean(JedisCluster.class) +@ConditionalOnCache("redis-cluster-jedis") +public class JedisClusterBucket4jConfiguration { + + public final JedisCluster jedisCluster; + private final String configCacheName; + + public JedisClusterBucket4jConfiguration(JedisCluster jedisCluster, Bucket4JBootProperties properties) { + this.jedisCluster = jedisCluster; + this.configCacheName = properties.getFilterConfigCacheName(); + } + + @Bean + @ConditionalOnMissingBean(SyncCacheResolver.class) + public SyncCacheResolver bucket4RedisResolver() { + return new JedisClusterCacheResolver(jedisCluster); + } + + @Bean + @ConditionalOnMissingBean(CacheManager.class) + @ConditionalOnFilterConfigCacheEnabled + public CacheManager configCacheManager() { + return new JedisClusterCacheManager<>(jedisCluster, configCacheName, Bucket4JConfiguration.class); + } + + @Bean + @ConditionalOnFilterConfigCacheEnabled + public JedisClusterCacheListener configCacheListener(ApplicationEventPublisher eventPublisher) { + return new JedisClusterCacheListener<>(jedisCluster, configCacheName, String.class, Bucket4JConfiguration.class, eventPublisher); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterCacheListener.java b/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterCacheListener.java new file mode 100644 index 000000000..4538521df --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterCacheListener.java @@ -0,0 +1,113 @@ +/** ****************************************************************************** + * Copyright (c) 2025 Precies. Software OU and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.cache; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.giffing.bucket4j.spring.boot.starter.config.cache.CacheUpdateEvent; +import io.micrometer.core.instrument.util.NamedThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; +import redis.clients.jedis.JedisCluster; +import redis.clients.jedis.JedisPubSub; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class JedisClusterCacheListener extends JedisPubSub { + + private static final Logger LOGGER = LoggerFactory.getLogger(JedisClusterCacheListener.class); + + private final JedisCluster jedisCluster; + private final ObjectMapper objectMapper = new ObjectMapper(); + private final String updateChannel; + private final JavaType deserializeType; + private final ApplicationEventPublisher eventPublisher; + + /** + * @param jedisCluster The cluster to use for listening/publishing events + * @param cacheName The name of the cache. This is used as prefix for the event channels + * @param keyType The type of the key. This is required for parsing events and should match the K of this class. + * @param valueType The type of the value. This is required for parsing events and should match the V of this class. + */ + public JedisClusterCacheListener(JedisCluster jedisCluster, String cacheName, Class keyType, Class valueType, ApplicationEventPublisher eventPublisher) { + this.jedisCluster = jedisCluster; + this.updateChannel = cacheName.concat(":update"); + this.deserializeType = objectMapper.getTypeFactory().constructParametricType(CacheUpdateEvent.class, keyType, valueType); + this.eventPublisher = eventPublisher; + subscribe(); + } + + public void subscribe() { + Thread thread = new Thread(() -> { + AtomicInteger reconnectBackoffTimeMillis = new AtomicInteger(1000); + // Using a NamedThreadFactory for creating a Daemon thread, so it will never block the jvm from closing. + ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("reset-reconnect-backoff-thread")); + ScheduledFuture resetTask = null; + + while(!Thread.currentThread().isInterrupted() && isUp()){ + try { + // Schedule a reset of the backoff after 10 seconds. + // This is done in a different thread since subscribe is a blocking call. + resetTask = executorService.schedule(()-> reconnectBackoffTimeMillis.set(1000), 10000, TimeUnit.MILLISECONDS); + + jedisCluster.subscribe(this, updateChannel); + } catch (Exception e) { + LOGGER.error("Failed to connect the Jedis subscriber, attempting to reconnect in {} seconds. " + + "Exception was: {}", (reconnectBackoffTimeMillis.get() /1000), e.getMessage()); + + // Cancel the reset of the backoff + if(resetTask != null) { + resetTask.cancel(true); + resetTask = null; + } + + // Wait before trying to reconnect and increase the backoff duration + try { + Thread.sleep(reconnectBackoffTimeMillis.get()); + // exponentially increase the backoff with a max of 30 seconds + reconnectBackoffTimeMillis.set(Math.min((reconnectBackoffTimeMillis.get() * 2), 30000)); + } catch (InterruptedException ignored) { + // ignored, already interrupted so the while loop will stop + } + } + } + }, "JedisSubscriberThread"); + thread.setDaemon(true); + thread.start(); + } + + private boolean isUp() { + return jedisCluster.ping().equals("PONG"); + } + + @Override + public void onMessage(String channel, String message) { + if (channel.equals(updateChannel)) { + onCacheUpdateEvent(message); + } else { + LOGGER.debug("Unsupported cache event received of type "); + } + } + + private void onCacheUpdateEvent(String message) { + try { + CacheUpdateEvent event = objectMapper.readValue(message, deserializeType); + this.eventPublisher.publishEvent(event); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterCacheManager.java b/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterCacheManager.java new file mode 100644 index 000000000..2d1ad5564 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterCacheManager.java @@ -0,0 +1,76 @@ +/** ****************************************************************************** + * Copyright (c) 2025 Precies. Software OU and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.cache; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.giffing.bucket4j.spring.boot.starter.config.cache.CacheManager; +import com.giffing.bucket4j.spring.boot.starter.config.cache.CacheUpdateEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import redis.clients.jedis.JedisCluster; + +public class JedisClusterCacheManager implements CacheManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(JedisClusterCacheManager.class); + + private final JedisCluster cluster; + private final String cacheName; + private final Class valueType; + private final ObjectMapper objectMapper; + private final String updateChannel; + + /** + * @param cluster The JedisCluster to use for reading/writing data to the cache + * @param cacheName The name of the cache. + * @param valueType The type of the data. This is required for parsing and should always match the V of this class. + */ + public JedisClusterCacheManager(JedisCluster cluster, String cacheName, Class valueType) { + this.cluster = cluster; + this.cacheName = cacheName; + this.valueType = valueType; + + this.objectMapper = new ObjectMapper(); + this.updateChannel = cacheName.concat(":update"); + } + + + @Override + public V getValue(K key) { + try { + String serializedValue = cluster.hget(cacheName, objectMapper.writeValueAsString(key)); + return serializedValue != null ? objectMapper.readValue(serializedValue, this.valueType) : null; + } catch (JsonProcessingException e) { + LOGGER.warn("Exception occurred while retrieving key '{}' from cache '{}'. Message: {}", key, cacheName, e.getMessage()); + return null; + } + } + + @Override + public void setValue(K key, V value) { + try { + V oldValue = getValue(key); + + String serializedKey = objectMapper.writeValueAsString(key); + String serializedValue = objectMapper.writeValueAsString(value); + cluster.hset(this.cacheName, serializedKey, serializedValue); + + //publish an update event if the key already existed + if(oldValue != null){ + CacheUpdateEvent updateEvent = new CacheUpdateEvent<>(key, oldValue, value); + cluster.publish(this.updateChannel, objectMapper.writeValueAsString(updateEvent)); + } + } catch (JsonProcessingException e) { + LOGGER.warn("Exception occurred while setting key '{}' in cache '{}'. Message: {}", key, cacheName, e.getMessage()); + throw new RuntimeException(e); + } + } +} + diff --git a/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterCacheResolver.java b/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterCacheResolver.java new file mode 100644 index 000000000..931b16bb6 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/cache/JedisClusterCacheResolver.java @@ -0,0 +1,47 @@ +/** ****************************************************************************** + * Copyright (c) 2025 Precies. Software OU and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.cache; + +import com.giffing.bucket4j.spring.boot.starter.config.cache.AbstractCacheResolverTemplate; +import com.giffing.bucket4j.spring.boot.starter.config.cache.SyncCacheResolver; +import io.github.bucket4j.distributed.ExpirationAfterWriteStrategy; +import io.github.bucket4j.distributed.proxy.AbstractProxyManager; +import io.github.bucket4j.redis.jedis.cas.JedisBasedProxyManager; +import redis.clients.jedis.JedisCluster; + +import java.time.Duration; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class JedisClusterCacheResolver extends AbstractCacheResolverTemplate implements SyncCacheResolver { + + private final JedisCluster jedisCluster; + + public JedisClusterCacheResolver(JedisCluster jedisCluster) { + this.jedisCluster = jedisCluster; + } + + @Override + public boolean isAsync() { + return false; + } + + @Override + public byte[] castStringToCacheKey(String key) { + return key.getBytes(UTF_8); + } + + @Override + public AbstractProxyManager getProxyManager(String cacheName) { + return JedisBasedProxyManager.builderFor(jedisCluster) + .withExpirationStrategy(ExpirationAfterWriteStrategy.basedOnTimeForRefillingBucketUpToMax(Duration.ofSeconds(10))) + .build(); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/entities/AuthToken.java b/server/src/main/java/org/eclipse/openvsx/entities/AuthToken.java index 032a9c5a9..f2ff6e261 100644 --- a/server/src/main/java/org/eclipse/openvsx/entities/AuthToken.java +++ b/server/src/main/java/org/eclipse/openvsx/entities/AuthToken.java @@ -11,7 +11,6 @@ import java.io.Serializable; import java.time.Instant; -import java.util.Objects; import java.util.Set; /** @@ -27,24 +26,6 @@ public record AuthToken( Instant refreshExpiresAt ) implements Serializable { - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AuthToken authToken = (AuthToken) o; - return Objects.equals(accessToken, authToken.accessToken) - && Objects.equals(issuedAt, authToken.issuedAt) - && Objects.equals(expiresAt, authToken.expiresAt) - && Objects.equals(scopes, authToken.scopes) - && Objects.equals(refreshToken, authToken.refreshToken) - && Objects.equals(refreshExpiresAt, authToken.refreshExpiresAt); - } - - @Override - public int hashCode() { - return Objects.hash(accessToken, issuedAt, expiresAt, scopes, refreshToken, refreshExpiresAt); - } - @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java b/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java index ddb128745..d8cf326f4 100644 --- a/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java +++ b/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java @@ -11,14 +11,8 @@ import jakarta.persistence.*; -import java.io.Serial; -import java.io.Serializable; - @Entity -public class FileResource implements Serializable { - - @Serial - private static final long serialVersionUID = 1L; +public class FileResource { // Resource types public static final String DOWNLOAD = "download"; diff --git a/server/src/main/java/org/eclipse/openvsx/json/BadgeJson.java b/server/src/main/java/org/eclipse/openvsx/json/BadgeJson.java index 35aa4c5bf..f15ce7630 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/BadgeJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/BadgeJson.java @@ -11,11 +11,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; - import io.swagger.v3.oas.annotations.media.Schema; -import java.io.Serial; -import java.io.Serializable; import java.util.Objects; @Schema( @@ -23,10 +20,7 @@ description = "A badge to be shown in the sidebar of the extension page in the registry" ) @JsonInclude(Include.NON_NULL) -public class BadgeJson implements Serializable { - - @Serial - private static final long serialVersionUID = 1L; +public class BadgeJson { @Schema(description = "Image URL of the badge") private String url; diff --git a/server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java b/server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java index 82a5f53bb..4285097ea 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/ChangeNamespaceJson.java @@ -11,8 +11,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; -import java.io.Serializable; - /** * Used to change a namespace */ @@ -22,4 +20,4 @@ public record ChangeNamespaceJson( String newNamespace, boolean removeOldNamespace, boolean mergeIfNewNamespaceAlreadyExists -) implements Serializable {} +) {} diff --git a/server/src/main/java/org/eclipse/openvsx/json/ExtensionJson.java b/server/src/main/java/org/eclipse/openvsx/json/ExtensionJson.java index ba7334255..0caa6ae80 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/ExtensionJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/ExtensionJson.java @@ -16,8 +16,6 @@ import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; -import java.io.Serial; -import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Objects; @@ -29,10 +27,7 @@ description = "Metadata of an extension" ) @JsonInclude(Include.NON_NULL) -public class ExtensionJson extends ResultJson implements Serializable { - - @Serial - private static final long serialVersionUID = 1L; +public class ExtensionJson extends ResultJson { public static ExtensionJson error(String message) { var info = new ExtensionJson(); diff --git a/server/src/main/java/org/eclipse/openvsx/json/ExtensionReferenceJson.java b/server/src/main/java/org/eclipse/openvsx/json/ExtensionReferenceJson.java index a8f654046..69335b6ee 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/ExtensionReferenceJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/ExtensionReferenceJson.java @@ -9,15 +9,11 @@ ********************************************************************************/ package org.eclipse.openvsx.json; -import jakarta.validation.constraints.NotNull; - import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; - import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotNull; -import java.io.Serial; -import java.io.Serializable; import java.util.Objects; @Schema( @@ -25,10 +21,7 @@ description = "A reference to another extension in the registry" ) @JsonInclude(Include.NON_NULL) -public class ExtensionReferenceJson implements Serializable { - - @Serial - private static final long serialVersionUID = 1L; +public class ExtensionReferenceJson { @Schema(description = "URL to get metadata of the referenced extension") @NotNull diff --git a/server/src/main/java/org/eclipse/openvsx/json/ExtensionReplacementJson.java b/server/src/main/java/org/eclipse/openvsx/json/ExtensionReplacementJson.java index 3f62dadff..5d75d7c14 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/ExtensionReplacementJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/ExtensionReplacementJson.java @@ -12,18 +12,12 @@ import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.v3.oas.annotations.media.Schema; -import java.io.Serial; -import java.io.Serializable; - @Schema( name = "ExtensionReplacement", description = "Metadata of an extension replacement" ) @JsonInclude(JsonInclude.Include.NON_NULL) -public class ExtensionReplacementJson implements Serializable { - - @Serial - private static final long serialVersionUID = 1L; +public class ExtensionReplacementJson { @Schema(description = "URL of the extension replacement") private String url; diff --git a/server/src/main/java/org/eclipse/openvsx/json/NamespaceDetailsJson.java b/server/src/main/java/org/eclipse/openvsx/json/NamespaceDetailsJson.java index 3f87d2980..974415ac5 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/NamespaceDetailsJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/NamespaceDetailsJson.java @@ -4,8 +4,6 @@ import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotNull; -import java.io.Serial; -import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Objects; @@ -15,10 +13,7 @@ description = "Details of a namespace" ) @JsonInclude(JsonInclude.Include.NON_NULL) -public class NamespaceDetailsJson extends ResultJson implements Serializable { - - @Serial - private static final long serialVersionUID = 1L; +public class NamespaceDetailsJson extends ResultJson { public static NamespaceDetailsJson error(String message) { var result = new NamespaceDetailsJson(); diff --git a/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java b/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java index 5015f04bc..ca0e8fc12 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/SearchEntryJson.java @@ -16,8 +16,6 @@ import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; -import java.io.Serial; -import java.io.Serializable; import java.util.List; import java.util.Map; @@ -26,10 +24,7 @@ description = "Summary of metadata of an extension" ) @JsonInclude(Include.NON_NULL) -public class SearchEntryJson implements Serializable { - - @Serial - private static final long serialVersionUID = 1L; +public class SearchEntryJson { @Schema(description = "URL to get the full metadata of the extension") @NotNull diff --git a/server/src/main/java/org/eclipse/openvsx/json/UserJson.java b/server/src/main/java/org/eclipse/openvsx/json/UserJson.java index f41844b60..ff3c17fee 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/UserJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/UserJson.java @@ -9,27 +9,20 @@ ********************************************************************************/ package org.eclipse.openvsx.json; -import java.io.Serial; -import java.io.Serializable; -import java.util.List; -import java.util.Objects; - -import jakarta.validation.constraints.NotNull; - import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; - import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotNull; + +import java.util.List; +import java.util.Objects; @Schema( name = "User", description = "User data" ) @JsonInclude(Include.NON_NULL) -public class UserJson extends ResultJson implements Serializable { - - @Serial - private static final long serialVersionUID = 1L; +public class UserJson extends ResultJson { public static UserJson error(String message) { var user = new UserJson(); @@ -149,10 +142,7 @@ public void setAdditionalLogins(List additionalLogins) { } @JsonInclude(Include.NON_NULL) - public static class PublisherAgreement implements Serializable { - - @Serial - private static final long serialVersionUID = 1L; + public static class PublisherAgreement { /* 'none' | 'signed' | 'outdated' */ private String status; diff --git a/server/src/main/java/org/eclipse/openvsx/json/VersionReferenceJson.java b/server/src/main/java/org/eclipse/openvsx/json/VersionReferenceJson.java index 304e4ec6a..a8e3799a9 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/VersionReferenceJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/VersionReferenceJson.java @@ -11,17 +11,15 @@ import io.swagger.v3.oas.annotations.media.Schema; -import java.io.Serializable; import java.util.Map; import static org.eclipse.openvsx.util.TargetPlatform.*; -import static org.eclipse.openvsx.util.TargetPlatform.NAME_UNIVERSAL; @Schema( name = "VersionReference", description = "Essential metadata of an extension version" ) -public class VersionReferenceJson implements Serializable { +public class VersionReferenceJson { @Schema(description = "URL to get the full metadata of this version") private String url; diff --git a/server/src/main/java/org/eclipse/openvsx/json/VersionTargetPlatformsJson.java b/server/src/main/java/org/eclipse/openvsx/json/VersionTargetPlatformsJson.java index 2fc2bb18c..ed11a8f1c 100644 --- a/server/src/main/java/org/eclipse/openvsx/json/VersionTargetPlatformsJson.java +++ b/server/src/main/java/org/eclipse/openvsx/json/VersionTargetPlatformsJson.java @@ -9,6 +9,4 @@ * ****************************************************************************** */ package org.eclipse.openvsx.json; -import java.io.Serializable; - -public record VersionTargetPlatformsJson(String version, String[] targetPlatforms) implements Serializable {} +public record VersionTargetPlatformsJson(String version, String[] targetPlatforms) {} diff --git a/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java b/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java index 45e94de47..745ad6f7b 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/DatabaseSearchService.java @@ -14,20 +14,19 @@ import org.eclipse.openvsx.entities.Extension; import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.RelevanceService.SearchStats; +import org.eclipse.openvsx.util.ErrorResultException; import org.eclipse.openvsx.util.TargetPlatform; import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; -import org.springframework.data.elasticsearch.core.SearchHit; -import org.springframework.data.elasticsearch.core.SearchHits; -import org.springframework.data.elasticsearch.core.SearchHitsImpl; -import org.springframework.data.elasticsearch.core.TotalHitsRelation; import org.springframework.data.util.Streamable; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; -import java.time.Duration; -import java.util.*; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Map; import java.util.stream.Stream; import static org.eclipse.openvsx.cache.CacheService.CACHE_AVERAGE_REVIEW_RATING; @@ -57,7 +56,7 @@ public boolean isEnabled() { @Transactional @Cacheable(CACHE_DATABASE_SEARCH) @CacheEvict(value = CACHE_AVERAGE_REVIEW_RATING, allEntries = true) - public SearchHits search(ISearchService.Options options) { + public SearchResult search(ISearchService.Options options) { var matchingExtensions = repositories.findAllActiveExtensions(); matchingExtensions = excludeByNamespace(options, matchingExtensions); matchingExtensions = excludeByTargetPlatform(options, matchingExtensions); @@ -67,11 +66,7 @@ public SearchHits search(ISearchService.Options options) { var sortedExtensions = sortExtensions(options, matchingExtensions); var totalHits = sortedExtensions.size(); sortedExtensions = applyPaging(options, sortedExtensions); - var searchHits = sortedExtensions.stream() - .map(extensionSearch -> new SearchHit<>(null, null, null, 0.0f, null, null, null, null, null, null, extensionSearch)) - .toList(); - - return new SearchHitsImpl<>(totalHits, TotalHitsRelation.OFF, 0f, Duration.ZERO, null, null, searchHits, null, null, null); + return new SearchResult(totalHits, sortedExtensions); } private List applyPaging(Options options, List sortedExtensions) { @@ -82,7 +77,7 @@ private List applyPaging(Options options, List private List sortExtensions(Options options, Streamable matchingExtensions) { Stream searchEntries; - if("relevance".equals(options.sortBy()) || "rating".equals(options.sortBy())) { + if(SortBy.RELEVANCE.equals(options.sortBy()) || SortBy.RATING.equals(options.sortBy())) { var searchStats = new SearchStats(repositories); searchEntries = matchingExtensions.stream().map(extension -> relevanceService.toSearchEntry(extension, searchStats)); } else { @@ -94,13 +89,16 @@ private List sortExtensions(Options options, Streamable search(Options options) { + public SearchResult search(Options options) { var resultWindow = options.requestedOffset() + options.requestedSize(); if(resultWindow > getMaxResultWindow()) { - return new SearchHitsImpl<>(0, TotalHitsRelation.OFF, 0f, Duration.ZERO, null, null, Collections.emptyList(), null, null, null); + return new SearchResult(0L, Collections.emptyList()); } var queryBuilder = new NativeQueryBuilder(); @@ -279,30 +281,19 @@ public SearchHits search(Options options) { } } + var firstSearchHitsPage = searchHitsList.get(0); + List> searchHits = new ArrayList<>(firstSearchHitsPage.getSearchHits()); if(searchHitsList.size() == 2) { - var firstSearchHitsPage = searchHitsList.get(0); var secondSearchHitsPage = searchHitsList.get(1); - List> searchHits = new ArrayList<>(firstSearchHitsPage.getSearchHits()); searchHits.addAll(secondSearchHitsPage.getSearchHits()); var endIndex = Math.min(searchHits.size(), options.requestedOffset() + options.requestedSize()); var startIndex = Math.min(endIndex, options.requestedOffset()); searchHits = searchHits.subList(startIndex, endIndex); - return new SearchHitsImpl<>( - firstSearchHitsPage.getTotalHits(), - firstSearchHitsPage.getTotalHitsRelation(), - firstSearchHitsPage.getMaxScore(), - Duration.ZERO, - null, - null, - searchHits, - null, - null, - null - ); - } else { - return searchHitsList.get(0); } + + var results = searchHits.stream().map(SearchHit::getContent).toList(); + return new SearchResult(firstSearchHitsPage.getTotalHits(), results); } private ObjectBuilder createSearchQuery(BoolQuery.Builder boolQuery, Options options) { @@ -369,20 +360,20 @@ private void sortResults(NativeQueryBuilder queryBuilder, String sortOrder, Stri } var types = Map.of( - "relevance", FieldType.Float, - "rating", FieldType.Float, - "timestamp", FieldType.Long, - "downloadCount", FieldType.Integer + SortBy.RELEVANCE, FieldType.Float, + SortBy.RATING, FieldType.Float, + SortBy.TIMESTAMP, FieldType.Long, + SortBy.DOWNLOADS, FieldType.Integer ); var type = types.get(sortBy); if(type == null) { - throw new ErrorResultException("sortBy parameter must be 'relevance', 'timestamp', 'averageRating' or 'downloadCount'."); + throw new ErrorResultException("sortBy parameter must be " + SortBy.OPTIONS + "."); } var scoreSort = new SortOptions.Builder().score(builder -> builder.order(order)).build(); var fieldSort = new SortOptions.Builder().field(builder -> builder.field(sortBy).unmappedType(type).order(order)).build(); - var sortOptions = sortBy.equals("relevance") ? List.of(scoreSort, fieldSort) : List.of(fieldSort, scoreSort); + var sortOptions = sortBy.equals(SortBy.RELEVANCE) ? List.of(scoreSort, fieldSort) : List.of(fieldSort, scoreSort); queryBuilder.withSort(sortOptions); } diff --git a/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java b/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java index 36b7fa82e..a03ceaf66 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java +++ b/server/src/main/java/org/eclipse/openvsx/search/ExtensionSearch.java @@ -9,22 +9,16 @@ ********************************************************************************/ package org.eclipse.openvsx.search; -import java.io.Serial; -import java.io.Serializable; -import java.util.List; -import java.util.Objects; - -import javax.annotation.Nullable; - import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; -@Document(indexName = "extensions") -public class ExtensionSearch implements Serializable { +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; - @Serial - private static final long serialVersionUID = 1L; +@Document(indexName = "extensions") +public class ExtensionSearch { @Field(index = false) private long id; diff --git a/server/src/main/java/org/eclipse/openvsx/search/ISearchService.java b/server/src/main/java/org/eclipse/openvsx/search/ISearchService.java index accb8f988..484a2041f 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/ISearchService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/ISearchService.java @@ -10,7 +10,6 @@ package org.eclipse.openvsx.search; import org.eclipse.openvsx.entities.Extension; -import org.springframework.data.elasticsearch.core.SearchHits; import java.util.Arrays; import java.util.Collection; @@ -30,7 +29,7 @@ public interface ISearchService { /** * Search with given options */ - SearchHits search(Options options); + SearchResult search(Options options); /** * Updating the search index has two modes: diff --git a/server/src/main/java/org/eclipse/openvsx/search/SearchResult.java b/server/src/main/java/org/eclipse/openvsx/search/SearchResult.java new file mode 100644 index 000000000..204c48c62 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/search/SearchResult.java @@ -0,0 +1,51 @@ +/** ****************************************************************************** + * Copyright (c) 2025 Precies. Software OU and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.search; + +import jakarta.validation.constraints.NotNull; + +import java.util.Collections; +import java.util.List; + +public class SearchResult { + + private long totalHits; + private List hits; + + public SearchResult() { + totalHits = 0; + hits = Collections.emptyList(); + } + + public SearchResult(long totalHits, @NotNull List hits) { + this.totalHits = totalHits; + this.hits = hits; + } + + public long getTotalHits() { + return totalHits; + } + + public void setTotalHits(long totalHits) { + this.totalHits = totalHits; + } + + public List getHits() { + return hits; + } + + public void setHits(List hits) { + this.hits = hits; + } + + public boolean hasSearchHits() { + return !hits.isEmpty(); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/search/SearchUtilService.java b/server/src/main/java/org/eclipse/openvsx/search/SearchUtilService.java index 16d99971c..e78309d99 100644 --- a/server/src/main/java/org/eclipse/openvsx/search/SearchUtilService.java +++ b/server/src/main/java/org/eclipse/openvsx/search/SearchUtilService.java @@ -58,7 +58,7 @@ protected ISearchService getImplementation() { } - public SearchHits search(ElasticSearchService.Options options) { + public SearchResult search(ElasticSearchService.Options options) { return getImplementation().search(options); } diff --git a/server/src/main/java/org/eclipse/openvsx/search/SortBy.java b/server/src/main/java/org/eclipse/openvsx/search/SortBy.java new file mode 100644 index 000000000..766984517 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/search/SortBy.java @@ -0,0 +1,18 @@ +/** ****************************************************************************** + * Copyright (c) 2025 Precies. Software OU and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.search; + +public class SortBy { + public static final String RELEVANCE = "relevance"; + public static final String TIMESTAMP = "timestamp"; + public static final String RATING = "rating"; + public static final String DOWNLOADS = "downloadCount"; + public static final String OPTIONS = String.format("'%s', '%s', '%s' or '%s'", RELEVANCE, TIMESTAMP, RATING, DOWNLOADS); +} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java index 78d2de095..e1aaf541e 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java @@ -247,7 +247,7 @@ private void copy(String oldObjectKey, String newObjectKey) { } @Override - @Cacheable(value = CACHE_EXTENSION_FILES, keyGenerator = GENERATOR_FILES) + @Cacheable(value = CACHE_EXTENSION_FILES, keyGenerator = GENERATOR_FILES, cacheManager = "fileCacheManager") public Path getCachedFile(FileResource resource) { var objectKey = getObjectKey(resource); var request = GetObjectRequest.builder() diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java index c6c0eedf4..dcb1259fe 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java @@ -236,7 +236,7 @@ public void copyNamespaceLogo(Namespace oldNamespace, Namespace newNamespace) { } @Override - @Cacheable(value = CACHE_EXTENSION_FILES, keyGenerator = GENERATOR_FILES) + @Cacheable(value = CACHE_EXTENSION_FILES, keyGenerator = GENERATOR_FILES, cacheManager = "fileCacheManager") public Path getCachedFile(FileResource resource) { var blobName = getBlobName(resource); if (StringUtils.isEmpty(serviceEndpoint)) { diff --git a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java index b93d58b6a..4738e8932 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java @@ -212,7 +212,7 @@ private void copy(String source, String target) { } @Override - @Cacheable(value = CACHE_EXTENSION_FILES, keyGenerator = GENERATOR_FILES) + @Cacheable(value = CACHE_EXTENSION_FILES, keyGenerator = GENERATOR_FILES, cacheManager = "fileCacheManager") public Path getCachedFile(FileResource resource) { if (StringUtils.isEmpty(bucketId)) { throw new IllegalStateException(missingBucketIdMessage(resource.getName())); diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java index be36fc48c..78cc504d6 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java @@ -9,6 +9,8 @@ ********************************************************************************/ package org.eclipse.openvsx.storage; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.common.collect.Maps; import jakarta.persistence.EntityManager; import jakarta.transaction.Transactional; @@ -20,12 +22,10 @@ import org.eclipse.openvsx.repositories.RepositoryService; import org.eclipse.openvsx.search.SearchUtilService; import org.eclipse.openvsx.util.TempFile; +import org.eclipse.openvsx.util.UrlUtil; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.util.Pair; -import org.springframework.http.CacheControl; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; +import org.springframework.http.*; import org.springframework.stereotype.Component; import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; @@ -315,6 +315,22 @@ public ResponseEntity getFileResponse(Path path) { }); } + public ResponseEntity getFileResponse(ArrayNode node) { + var baseUrl = UrlUtil.getBaseUrl(); + var headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + return ResponseEntity.ok() + .headers(headers) + .body(outputStream -> { + var mapper = new ObjectMapper(); + var value = mapper.createArrayNode(); + for(var item : node) { + value.add(baseUrl + item.asText()); + } + mapper.writeValue(outputStream, value); + }); + } + public ResponseEntity getNamespaceLogo(Namespace namespace) { if (namespace.getLogoStorageType().equals(STORAGE_LOCAL)) { return localStorage.getNamespaceLogo(namespace); diff --git a/server/src/main/resources/ehcache.xml b/server/src/main/resources/ehcache.xml deleted file mode 100644 index 751dd1517..000000000 --- a/server/src/main/resources/ehcache.xml +++ /dev/null @@ -1,121 +0,0 @@ - - - - - - - - 1 - 1 - 2 - - - - - 1 - - - 1024 - 32 - 128 - - - - - 1 - - - 1024 - - - - - 1 - - - 1024 - 32 - 128 - - - - - 1 - - - 1024 - 32 - 128 - - - - - 1 - - - 1024 - 32 - 128 - - - - - 1 - - - 1 - 2 - 8 - - - - - 1 - - - 1 - 2 - 8 - - - - - 1 - - - - org.eclipse.openvsx.cache.ExpiredFileListener - ASYNCHRONOUS - UNORDERED - EXPIRED - EVICTED - REMOVED - UPDATED - - - - 150 - - - - - 1 - - - - org.eclipse.openvsx.cache.ExpiredFileListener - ASYNCHRONOUS - UNORDERED - EXPIRED - EVICTED - REMOVED - UPDATED - - - - 20 - - - \ No newline at end of file diff --git a/server/src/test/java/org/eclipse/openvsx/IntegrationTest.java b/server/src/test/java/org/eclipse/openvsx/IntegrationTest.java index 630301d39..ca6e12f0f 100644 --- a/server/src/test/java/org/eclipse/openvsx/IntegrationTest.java +++ b/server/src/test/java/org/eclipse/openvsx/IntegrationTest.java @@ -11,7 +11,6 @@ import com.fasterxml.jackson.databind.JsonNode; import org.eclipse.openvsx.json.*; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,14 +23,11 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.web.client.RestTemplate; -import javax.cache.CacheManager; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import static org.assertj.core.api.Assertions.assertThat; -import static org.eclipse.openvsx.cache.CacheService.CACHE_EXTENSION_FILES; -import static org.eclipse.openvsx.cache.CacheService.CACHE_WEB_RESOURCE_FILES; @SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT) @ActiveProfiles("test") @@ -55,15 +51,6 @@ private String apiCall(String path) { return "http://localhost:" + port + path; } - @AfterAll - static void clearFileCaches(@Autowired CacheManager cacheManager) { - var cache = cacheManager.getCache(CACHE_WEB_RESOURCE_FILES); - cache.removeAll(); - - cache = cacheManager.getCache(CACHE_EXTENSION_FILES); - cache.removeAll(); - } - @Test void testPublishExtension() throws Exception { testService.createUser(); diff --git a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java index 5ed086552..89aec2767 100644 --- a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java @@ -27,9 +27,7 @@ import org.eclipse.openvsx.publish.PublishExtensionVersionHandler; import org.eclipse.openvsx.publish.PublishExtensionVersionService; import org.eclipse.openvsx.repositories.RepositoryService; -import org.eclipse.openvsx.search.ExtensionSearch; -import org.eclipse.openvsx.search.ISearchService; -import org.eclipse.openvsx.search.SearchUtilService; +import org.eclipse.openvsx.search.*; import org.eclipse.openvsx.security.OAuth2AttributesConfig; import org.eclipse.openvsx.security.OAuth2UserServices; import org.eclipse.openvsx.security.SecurityConfig; @@ -52,9 +50,6 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; -import org.springframework.data.elasticsearch.core.SearchHit; -import org.springframework.data.elasticsearch.core.SearchHitsImpl; -import org.springframework.data.elasticsearch.core.TotalHitsRelation; import org.springframework.data.util.Streamable; import org.springframework.http.MediaType; import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository; @@ -68,7 +63,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Duration; import java.time.LocalDateTime; import java.util.*; import java.util.function.BiFunction; @@ -2184,13 +2178,12 @@ private List mockSearch() { extension.setId(1L); var entry1 = new ExtensionSearch(); entry1.setId(1); - var searchHit = new SearchHit<>("0", "1", null, 1.0f, null, null, null, null, null, null, entry1); - var searchHits = new SearchHitsImpl<>(1, TotalHitsRelation.EQUAL_TO, 1.0f, Duration.ZERO, null, null, List.of(searchHit), null, null, null); Mockito.when(search.isEnabled()) .thenReturn(true); - var searchOptions = new ISearchService.Options("foo", null, null, 10, 0, "desc", "relevance", false, null); + var searchResult = new SearchResult(1, List.of(entry1)); + var searchOptions = new ISearchService.Options("foo", null, null, 10, 0, "desc", SortBy.RELEVANCE, false, null); Mockito.when(search.search(searchOptions)) - .thenReturn(searchHits); + .thenReturn(searchResult); return List.of(extVersion); } diff --git a/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java b/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java index 62a985152..64e5cf51b 100644 --- a/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java @@ -24,9 +24,7 @@ import org.eclipse.openvsx.entities.*; import org.eclipse.openvsx.publish.ExtensionVersionIntegrityService; import org.eclipse.openvsx.repositories.RepositoryService; -import org.eclipse.openvsx.search.ExtensionSearch; -import org.eclipse.openvsx.search.ISearchService; -import org.eclipse.openvsx.search.SearchUtilService; +import org.eclipse.openvsx.search.*; import org.eclipse.openvsx.security.OAuth2AttributesConfig; import org.eclipse.openvsx.security.OAuth2UserServices; import org.eclipse.openvsx.security.SecurityConfig; @@ -42,9 +40,6 @@ import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; -import org.springframework.data.elasticsearch.core.SearchHit; -import org.springframework.data.elasticsearch.core.SearchHitsImpl; -import org.springframework.data.elasticsearch.core.TotalHitsRelation; import org.springframework.data.util.Streamable; import org.springframework.http.MediaType; import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository; @@ -59,7 +54,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.time.Duration; import java.time.LocalDateTime; import java.util.*; import java.util.stream.Collectors; @@ -630,19 +624,18 @@ private Extension mockSearch(String targetPlatform, String namespaceName, boolea var builtInExtensionNamespace = "vscode"; var entry1 = new ExtensionSearch(); entry1.setId(1); - List> searchResults = !builtInExtensionNamespace.equals(namespaceName) - ? Collections.singletonList(new SearchHit<>("0", "1", null, 1.0f, null, null, null, null, null, null, entry1)) + List searchHits = !builtInExtensionNamespace.equals(namespaceName) + ? Collections.singletonList(entry1) : Collections.emptyList(); - var searchHits = new SearchHitsImpl<>(searchResults.size(), TotalHitsRelation.EQUAL_TO, 1.0f, Duration.ZERO, null, null, - searchResults, null, null, null); + var searchResult = new SearchResult(searchHits.size(), searchHits); Mockito.when(integrityService.isEnabled()) .thenReturn(true); Mockito.when(search.isEnabled()) .thenReturn(true); - var searchOptions = new ISearchService.Options("yaml", null, targetPlatform, 50, 0, "desc", "relevance", false, new String[]{builtInExtensionNamespace}); + var searchOptions = new ISearchService.Options("yaml", null, targetPlatform, 50, 0, "desc", SortBy.RELEVANCE, false, new String[]{builtInExtensionNamespace}); Mockito.when(search.search(searchOptions)) - .thenReturn(searchHits); + .thenReturn(searchResult); var extension = mockExtension(); List results = active ? List.of(extension) : Collections.emptyList(); diff --git a/server/src/test/java/org/eclipse/openvsx/search/DatabaseSearchServiceTest.java b/server/src/test/java/org/eclipse/openvsx/search/DatabaseSearchServiceTest.java index 4a030d9f5..5396026a8 100644 --- a/server/src/test/java/org/eclipse/openvsx/search/DatabaseSearchServiceTest.java +++ b/server/src/test/java/org/eclipse/openvsx/search/DatabaseSearchServiceTest.java @@ -21,7 +21,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; -import org.springframework.data.elasticsearch.core.SearchHit; import org.springframework.data.util.Streamable; import org.springframework.test.context.bean.override.mockito.MockitoBean; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -63,12 +62,12 @@ void testRelevance() { var ext3 = mockExtension("openshift", 1.0, 100, 10, "redhat", List.of("Snippets", "Other")); Mockito.when(repositories.findAllActiveExtensions()).thenReturn(Streamable.of(List.of(ext1, ext2, ext3))); - var searchOptions = searchOptions(null, null, 50, 0, null, "relevance"); + var searchOptions = searchOptions(null, null, 50, 0, null, SortBy.RELEVANCE); var result = search.search(searchOptions); // should find all extensions but order should be different assertThat(result.getTotalHits()).isEqualTo(3); - var hits = result.getSearchHits(); + var hits = result.getHits(); // java should have the most relevance assertThat(getIdFromExtensionHits(hits, 0)).isEqualTo(getIdFromExtensionName("openshift")); assertThat(getIdFromExtensionHits(hits, 1)).isEqualTo(getIdFromExtensionName("yaml")); @@ -86,7 +85,7 @@ void testReverse() { // should find two extensions assertThat(result.getTotalHits()).isEqualTo(2); - var hits = result.getSearchHits(); + var hits = result.getHits(); assertThat(getIdFromExtensionHits(hits, 0)).isEqualTo(getIdFromExtensionName("java")); assertThat(getIdFromExtensionHits(hits, 1)).isEqualTo(getIdFromExtensionName("yaml")); } @@ -109,7 +108,7 @@ void testSimplePageSize() { // 7 total hits assertThat(result.getTotalHits()).isEqualTo(7); // but as we limit the page size it should only contains 5 - var hits = result.getSearchHits(); + var hits = result.getHits(); assertThat(hits.size()).isEqualTo(pageSizeItems); assertThat(getIdFromExtensionHits(hits, 0)).isEqualTo(getIdFromExtensionName("ext1")); @@ -137,7 +136,7 @@ void testPages() { // 7 total hits assertThat(result.getTotalHits()).isEqualTo(7); // But it should only contains 2 search items as specified by the pageSize - var hits = result.getSearchHits(); + var hits = result.getHits(); assertThat(hits.size()).isEqualTo(pageSizeItems); assertThat(getIdFromExtensionHits(hits, 0)).isEqualTo(getIdFromExtensionName("ext5")); @@ -158,7 +157,7 @@ void testQueryStringPublisherName() { assertThat(result.getTotalHits()).isEqualTo(3); // Check it found the correct extension - var hits = result.getSearchHits(); + var hits = result.getHits(); assertThat(getIdFromExtensionHits(hits, 0)).isEqualTo(getIdFromExtensionName("yaml")); assertThat(getIdFromExtensionHits(hits, 1)).isEqualTo(getIdFromExtensionName("java")); assertThat(getIdFromExtensionHits(hits, 2)).isEqualTo(getIdFromExtensionName("openshift")); @@ -178,7 +177,7 @@ void testQueryStringExtensionName() { assertThat(result.getTotalHits()).isEqualTo(1); // Check it found the correct extension - var hits = result.getSearchHits(); + var hits = result.getHits(); assertThat(getIdFromExtensionHits(hits, 0)).isEqualTo(getIdFromExtensionName("openshift")); } @@ -198,7 +197,7 @@ void testQueryStringDescription() { assertThat(result.getTotalHits()).isEqualTo(1); // Check it found the correct extension - var hits = result.getSearchHits(); + var hits = result.getHits(); assertThat(getIdFromExtensionHits(hits, 0)).isEqualTo(getIdFromExtensionName("openshift")); } @@ -219,7 +218,7 @@ void testQueryStringDisplayName() { assertThat(result.getTotalHits()).isEqualTo(1); // Check it found the correct extension - var hits = result.getSearchHits(); + var hits = result.getHits(); assertThat(getIdFromExtensionHits(hits, 0)).isEqualTo(getIdFromExtensionName("java")); } @@ -235,13 +234,13 @@ void testSortByTimeStamp() { ext4.getVersions().get(0).setTimestamp(LocalDateTime.parse("2021-10-06T00:00")); Mockito.when(repositories.findAllActiveExtensions()).thenReturn(Streamable.of(List.of(ext1, ext2, ext3, ext4))); - var searchOptions = searchOptions(null, null, 50, 0, null, "timestamp"); + var searchOptions = searchOptions(null, null, 50, 0, null, SortBy.TIMESTAMP); var result = search.search(searchOptions); // all extensions should be there assertThat(result.getTotalHits()).isEqualTo(4); // test now the order - var hits = result.getSearchHits(); + var hits = result.getHits(); assertThat(getIdFromExtensionHits(hits, 0)).isEqualTo(getIdFromExtensionName("foo")); assertThat(getIdFromExtensionHits(hits, 1)).isEqualTo(getIdFromExtensionName("java")); assertThat(getIdFromExtensionHits(hits, 2)).isEqualTo(getIdFromExtensionName("yaml")); @@ -256,13 +255,13 @@ void testSortByDownloadCount() { var ext4 = mockExtension("foo", 4.0, 100, 500, "bar", List.of("Other")); Mockito.when(repositories.findAllActiveExtensions()).thenReturn(Streamable.of(List.of(ext1, ext2, ext3, ext4))); - var searchOptions = searchOptions(null, null, 50, 0, null, "downloadCount"); + var searchOptions = searchOptions(null, null, 50, 0, null, SortBy.DOWNLOADS); var result = search.search(searchOptions); // all extensions should be there assertThat(result.getTotalHits()).isEqualTo(4); // test now the order - var hits = result.getSearchHits(); + var hits = result.getHits(); assertThat(getIdFromExtensionHits(hits, 0)).isEqualTo(getIdFromExtensionName("yaml")); assertThat(getIdFromExtensionHits(hits, 1)).isEqualTo(getIdFromExtensionName("openshift")); assertThat(getIdFromExtensionHits(hits, 2)).isEqualTo(getIdFromExtensionName("foo")); @@ -277,13 +276,13 @@ void testSortByRating() { var ext4 = mockExtension("foo", 1.0, 1, 0, "bar", List.of("Other")); Mockito.when(repositories.findAllActiveExtensions()).thenReturn(Streamable.of(List.of(ext1, ext2, ext3, ext4))); - var searchOptions = searchOptions(null, null, 50, 0, null, "rating"); + var searchOptions = searchOptions(null, null, 50, 0, null, SortBy.RATING); var result = search.search(searchOptions); // all extensions should be there assertThat(result.getTotalHits()).isEqualTo(4); // test now the order - var hits = result.getSearchHits(); + var hits = result.getHits(); assertThat(getIdFromExtensionHits(hits, 0)).isEqualTo(getIdFromExtensionName("foo")); assertThat(getIdFromExtensionHits(hits, 1)).isEqualTo(getIdFromExtensionName("openshift")); assertThat(getIdFromExtensionHits(hits, 2)).isEqualTo(getIdFromExtensionName("yaml")); @@ -307,7 +306,7 @@ private ISearchService.Options searchOptions( requestedOffset = 0; } if(sortBy == null) { - sortBy = "relevance"; + sortBy = SortBy.RELEVANCE; } return new ISearchService.Options( @@ -323,8 +322,8 @@ private ISearchService.Options searchOptions( ); } - long getIdFromExtensionHits(List> hits, int index) { - return hits.get(index).getContent().getId(); + long getIdFromExtensionHits(List hits, int index) { + return hits.get(index).getId(); } long getIdFromExtensionName(String extensionName) { diff --git a/server/src/test/java/org/eclipse/openvsx/search/ElasticSearchServiceTest.java b/server/src/test/java/org/eclipse/openvsx/search/ElasticSearchServiceTest.java index 355754d68..efdced8d6 100644 --- a/server/src/test/java/org/eclipse/openvsx/search/ElasticSearchServiceTest.java +++ b/server/src/test/java/org/eclipse/openvsx/search/ElasticSearchServiceTest.java @@ -176,7 +176,7 @@ void testSearchResultWindowTooLarge() { var options = new ISearchService.Options("foo", "bar", "universal", 50, 10000, null, null, false, null); var searchHits = search.search(options); - assertThat(searchHits.getSearchHits()).isEmpty(); + assertThat(searchHits.getHits()).isEmpty(); assertThat(searchHits.getTotalHits()).isZero(); } diff --git a/server/src/test/resources/application.yml b/server/src/test/resources/application.yml index ee342bc79..d5a207a03 100644 --- a/server/src/test/resources/application.yml +++ b/server/src/test/resources/application.yml @@ -1,7 +1,4 @@ spring: - cache: - jcache: - config: classpath:ehcache.xml datasource: driver-class-name: org.testcontainers.jdbc.ContainerDatabaseDriver url: jdbc:tc:postgresql:12.7:///test @@ -42,6 +39,8 @@ org: allow-anonymous-data-usage: false ovsx: + redis: + embedded: true storage: local: directory: /tmp