Redis Caching

docker-compose.yml

@@ -47,6 +47,157 @@ services:
     profiles:
       - kibana
 
+  redis-node-1:
+    image: redis:7.2
+    container_name: redis-node-1
+    ports:
+      - "7001:7000"
+    command: redis-server /etc/redis.conf
+    volumes:
+      - ./redis.conf:/etc/redis.conf
+      - /redis/node-1:/data
+    environment:
+      - REDISCLI_AUTH=openvsx
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
+      interval: 5s
+      retries: 5
+    profiles:
+      - redis
+
+  redis-node-2:
+    image: redis:7.2
+    container_name: redis-node-2
+    depends_on:
+      redis-node-1:
+        condition: service_healthy
+    ports:
+      - "7002:7000"
+    command: redis-server /etc/redis.conf
+    volumes:
+      - ./redis.conf:/etc/redis.conf
+      - /redis/node-2:/data
+    environment:
+      - REDISCLI_AUTH=openvsx
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
+      interval: 5s
+      retries: 5
+    profiles:
+      - redis
+
+  redis-node-3:
+    image: redis:7.2
+    container_name: redis-node-3
+    depends_on:
+      redis-node-2:
+        condition: service_healthy
+    ports:
+      - "7003:7000"
+    command: redis-server /etc/redis.conf
+    volumes:
+      - ./redis.conf:/etc/redis.conf
+      - /redis/node-3:/data
+    environment:
+      - REDISCLI_AUTH=openvsx
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
+      interval: 5s
+      retries: 5
+    profiles:
+      - redis
+
+  redis-node-4:
+    image: redis:7.2
+    container_name: redis-node-4
+    depends_on:
+      redis-node-3:
+        condition: service_healthy
+    ports:
+      - "7004:7000"
+    command: redis-server /etc/redis.conf
+    volumes:
+      - ./redis.conf:/etc/redis.conf
+      - /redis/node-4:/data
+    environment:
+      - REDISCLI_AUTH=openvsx
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
+      interval: 5s
+      retries: 5
+    profiles:
+      - redis
+
+  redis-node-5:
+    image: redis:7.2
+    container_name: redis-node-5
+    depends_on:
+      redis-node-4:
+        condition: service_healthy
+    ports:
+      - "7005:7000"
+    command: redis-server /etc/redis.conf
+    volumes:
+      - ./redis.conf:/etc/redis.conf
+      - /redis/node-5:/data
+    environment:
+      - REDISCLI_AUTH=openvsx
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
+      interval: 5s
+      retries: 5
+    profiles:
+      - redis
+
+  redis-node-6:
+    image: redis:7.2
+    container_name: redis-node-6
+    depends_on:
+      redis-node-5:
+        condition: service_healthy
+    ports:
+      - "7006:7000"
+    command: redis-server /etc/redis.conf
+    volumes:
+      - ./redis.conf:/etc/redis.conf
+      - /redis/node-6:/data
+    environment:
+      - REDISCLI_AUTH=openvsx
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "7000", "--user", "openvsx", "ping"]
+      interval: 5s
+      retries: 5
+    profiles:
+      - redis
+
+  redis-cluster-init:
+    image: redis:7.2
+    depends_on:
+      - redis-node-1
+      - redis-node-2
+      - redis-node-3
+      - redis-node-4
+      - redis-node-5
+      - redis-node-6
+    environment:
+      - REDISCLI_AUTH=openvsx
+    entrypoint: >
+      bash -c "
+      sleep 10;
+      echo yes | redis-cli --user openvsx --cluster create 
+      redis-node-1:7000 redis-node-2:7000 redis-node-3:7000 
+      redis-node-4:7000 redis-node-5:7000 redis-node-6:7000 
+      --cluster-replicas 1"
+    profiles:
+      - redis
+
+  redisinsight:
+    image: redis/redisinsight
+    ports:
+      - '5540:5540'
+    profiles:
+      - redisinsight
+
   server:
     image: openjdk:17
     working_dir: /app

redis.conf

@@ -0,0 +1,12 @@
+# Configuration for Redis nodes in docker-compose.yml
+port 7000
+cluster-enabled yes
+cluster-config-file nodes.conf
+cluster-node-timeout 5000
+appendonly yes
+maxmemory 64mb
+maxmemory-policy allkeys-lru
+user default off
+user openvsx on >openvsx ~* +@all
+masteruser openvsx
+masterauth openvsx

server/build.gradle

@@ -34,14 +34,15 @@ def versions = [
     woodstox: '6.4.0',
     jobrunr: '7.5.0',
     bucket4j: '0.12.7',
-    ehcache: '3.10.8',
+    bucket4j_redis: '8.10.1',
     tika: '3.1.0',
     bouncycastle: '1.80',
     commons_lang3: '3.12.0',
     jaxb_api: '2.3.1',
     jaxb_impl: '2.3.8',
-    gatling: '3.13.5',
-    loki4j: '1.4.2'
+    gatling: '3.9.5',
+    loki4j: '1.4.2',
+    embedded_redis: '1.4.3'
 ]
 ext['junit-jupiter.version'] = versions.junit
 sourceCompatibility = versions.java
@@ -80,6 +81,7 @@ dependencies {
     implementation "org.springframework.boot:spring-boot-starter-jooq"
     implementation "org.springframework.boot:spring-boot-starter-data-jpa"
     implementation "org.springframework.boot:spring-boot-starter-data-elasticsearch"
+    implementation "org.springframework.boot:spring-boot-starter-data-redis"
     implementation "org.springframework.boot:spring-boot-starter-security"
     implementation "org.springframework.boot:spring-boot-starter-actuator"
     implementation "org.springframework.boot:spring-boot-starter-cache"
@@ -89,8 +91,9 @@ dependencies {
     implementation "org.springframework.session:spring-session-jdbc"
     implementation "org.springframework.retry:spring-retry"
     implementation "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
-    implementation "org.ehcache:ehcache:${versions.ehcache}"
+    implementation "com.github.ben-manes.caffeine:caffeine"
     implementation "com.giffing.bucket4j.spring.boot.starter:bucket4j-spring-boot-starter:${versions.bucket4j}"
+    implementation "com.bucket4j:bucket4j-redis:${versions.bucket4j_redis}"
     implementation "org.jobrunr:jobrunr-spring-boot-3-starter:${versions.jobrunr}"
     implementation "org.flywaydb:flyway-core:${versions.flyway}"
     implementation "com.google.cloud:google-cloud-storage:${versions.gcloud}"
@@ -112,6 +115,7 @@ dependencies {
     implementation "io.micrometer:micrometer-tracing"
     implementation "io.micrometer:micrometer-tracing-bridge-otel"
     implementation "io.opentelemetry:opentelemetry-exporter-zipkin"
+    implementation "com.github.codemonstur:embedded-redis:${versions.embedded_redis}"
     runtimeOnly "io.micrometer:micrometer-registry-prometheus"
     runtimeOnly "org.postgresql:postgresql"
     jooqGenerator "org.postgresql:postgresql"

server/src/dev/resources/application.yml

@@ -16,9 +16,16 @@ spring:
     exclude: org.springframework.boot.actuate.autoconfigure.tracing.zipkin.ZipkinAutoConfiguration
   profiles:
     include: ovsx
-  cache:
-    jcache:
-      config: classpath:ehcache.xml
+  data:
+    redis:
+      # redis standalone configuration
+      host: localhost
+      port: 6379
+      # connect to redis cluster configured in docker-compose.yml
+#      cluster:
+#        nodes: '127.0.0.1:7001,127.0.0.1:7002,127.0.0.1:7003,127.0.0.1:7004,127.0.0.1:7005,127.0.0.1:7006'
+#      username: openvsx
+#      password: openvsx
   datasource:
     url: jdbc:postgresql://localhost:5432/postgres
     username: gitpod
@@ -101,6 +108,7 @@ org:
 
 bucket4j:
   enabled: true
+  cache-to-use: redis-jedis # use redis-cluster-jedis when running redis cluster
   filters:
     - cache-name: buckets
       url: '/api/-/(namespace/create|publish)'
@@ -140,7 +148,10 @@ ovsx:
   databasesearch:
     enabled: false
   elasticsearch:
+    enabled: true
     clear-on-start: true
+  redis:
+    embedded: true
   eclipse:
     base-url: https://api.eclipse.org
     publisher-agreement:

server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala

@@ -318,7 +318,7 @@ object Scenarios {
     val categories = Array("", "Programming Languages", "Themes", "Snippets", "Debuggers", "Linters", "Other")
     val sizes = Array("", "5", "500")
     val offsets = Array("", "1", "100", "25000")
-    val sortBys = Array("", "relevance", "timestamp", "averageRating", "downloadCount")
+    val sortBys = Array("", "relevance", "timestamp", "rating", "downloadCount")
     val sortOrders = Array("", "asc", "desc")
     val includeAllVersions = Array("", "true", "false")
 

server/src/main/java/org/eclipse/openvsx/LocalRegistryService.java

@@ -21,6 +21,7 @@
 import org.eclipse.openvsx.repositories.RepositoryService;
 import org.eclipse.openvsx.search.ExtensionSearch;
 import org.eclipse.openvsx.search.ISearchService;
+import org.eclipse.openvsx.search.SearchResult;
 import org.eclipse.openvsx.search.SearchUtilService;
 import org.eclipse.openvsx.storage.StorageUtilService;
 import org.eclipse.openvsx.util.*;
@@ -29,7 +30,6 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.data.domain.PageRequest;
-import org.springframework.data.elasticsearch.core.SearchHits;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Component;
@@ -257,11 +257,11 @@ public SearchResultJson search(ISearchService.Options options) {
             return json;
         }
 
-        var searchHits = search.search(options);
-        if(searchHits.hasSearchHits()) {
-            json.setExtensions(toSearchEntries(searchHits, options));
+        var result = search.search(options);
+        if(result.hasSearchHits()) {
+            json.setExtensions(toSearchEntries(result, options));
             json.setOffset(options.requestedOffset());
-            json.setTotalSize((int) searchHits.getTotalHits());
+            json.setTotalSize((int) result.getTotalHits());
         } else {
             json.setExtensions(Collections.emptyList());
         }
@@ -724,9 +724,9 @@ public ResultJson deleteReview(String namespace, String extensionName) {
         return ResultJson.success("Deleted review for " + NamingUtil.toExtensionId(extension));
     }
 
-    private LinkedHashMap<Long, ExtensionVersion> getLatestVersions(SearchHits<ExtensionSearch> searchHits) {
-        var ids = searchHits.stream()
-                .map(searchHit -> searchHit.getContent().getId())
+    private LinkedHashMap<Long, ExtensionVersion> getLatestVersions(SearchResult result) {
+        var ids = result.getHits().stream()
+                .map(ExtensionSearch::getId)
                 .distinct()
                 .collect(Collectors.toList());
 
@@ -756,9 +756,9 @@ private LinkedHashMap<Long, ExtensionVersion> findLatestVersions(Collection<Long
     }
 
 
-    private List<SearchEntryJson> toSearchEntries(SearchHits<ExtensionSearch> searchHits, ISearchService.Options options) {
+    private List<SearchEntryJson> toSearchEntries(SearchResult result, ISearchService.Options options) {
         var serverUrl = UrlUtil.getBaseUrl();
-        var latestVersions = getLatestVersions(searchHits);
+        var latestVersions = getLatestVersions(result);
         var membershipsByNamespaceId = getMemberships(latestVersions.values());
         var searchEntries = latestVersions.entrySet().stream()
                 .map(e -> {

server/src/main/java/org/eclipse/openvsx/RegistryAPI.java

@@ -22,6 +22,7 @@
 import org.eclipse.openvsx.entities.SemanticVersion;
 import org.eclipse.openvsx.json.*;
 import org.eclipse.openvsx.search.ISearchService;
+import org.eclipse.openvsx.search.SortBy;
 import org.eclipse.openvsx.util.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -757,8 +758,8 @@ public ResponseEntity<SearchResultJson> search(
             @RequestParam(defaultValue = "desc") 
             @Parameter(description = "Descending or ascending sort order", schema = @Schema(type = "string", allowableValues = {"asc", "desc"}))
             String sortOrder,
-            @RequestParam(defaultValue = "relevance")
-            @Parameter(description = "Sort key (relevance is a weighted mix of various properties)", schema = @Schema(type = "string", allowableValues = {"relevance", "timestamp", "averageRating", "downloadCount"}))
+            @RequestParam(defaultValue = SortBy.RELEVANCE)
+            @Parameter(description = "Sort key (relevance is a weighted mix of various properties)", schema = @Schema(type = "string", allowableValues = {SortBy.RELEVANCE, SortBy.TIMESTAMP, SortBy.RATING, SortBy.DOWNLOADS}))
             String sortBy,
             @RequestParam(defaultValue = "false")
             @Parameter(description = "Whether to include information on all available versions for each returned entry")

server/src/main/java/org/eclipse/openvsx/RegistryApplication.java

@@ -35,7 +35,6 @@
 @EnableScheduling
 @EnableRetry
 @EnableAsync
-@EnableCaching(proxyTargetClass = true)
 @EnableConfigurationProperties(OAuth2AttributesConfig.class)
 public class RegistryApplication {