cf: added support for individual mod/plugin files (#277)

DEVELOPMENT.md

@@ -6,28 +6,3 @@ Beyond the unit tests, ad hoc "integration testing" can be done by running via G
 ./gradlew run --args="assert fileExists build.gradle"
 ```
 
-## Creating a new release
-
-The Github Actions workflow `publish-release` will take of performing the release build, but the tag needs to be created with the release tasks shown below.
-
-### Patch release
-
-```shell
-./gradlew release -PpushReleaseTag=true
-```
-
-### Minor release
-
-```shell
-./gradlew releaseMinorVersion -PpushReleaseTag=true
-```
-
-## Creating a pre-release
-
-When needing to test a pre-release, such as a PR's changes, a pre-release can be run locally.
-
-Ensure `$HOME/.jreleaser/config.properties` contains `JRELEASER_GITHUB_TOKEN` set with a token with repo access.
-
-Invoke the gradle task `jreleaseRelease`.
-
-The release and artifacts are located at <https://github.com/itzg/mc-image-helper/releases/tag/early-access>

dev/curseforge.http

@@ -6,15 +6,26 @@ x-api-key: {{cfApiKey}}
 <> 2023-03-30T083426.200.json
 
 ###
-GET https://api.curseforge.com/v1/mods/350464
+# mod
+GET https://api.curseforge.com/v1/mods/search?gameId=432&slug=jei&classId=6
+x-api-key: {{cfApiKey}}
+
+###
+# mod
+GET https://api.curseforge.com/v1/mods/238222
+x-api-key: {{cfApiKey}}
+
+###
+# bukkit-plugin
+GET https://api.curseforge.com/v1/mods/31043
 x-api-key: {{cfApiKey}}
 
 ###
 GET https://api.curseforge.com/v1/mods/694605/files/4098018/download-url
 x-api-key: {{cfApiKey}}
 
 ###
-GET https://api.curseforge.com/v1/mods/369096/files/4560441
+GET https://api.curseforge.com/v1/mods/238222/files/4644453
 x-api-key: {{cfApiKey}}
 
 <> 2023-04-01T192357.200.json
@@ -31,4 +42,9 @@ x-api-key: {{cfApiKey}}
 GET https://api.curseforge.com/v1/mods/707734/files/4415193
 x-api-key: {{cfApiKey}}
 
-<> 2023-03-30T080528.403.html
+<> 2023-03-30T080528.403.html
+
+
+###
+GET http://localhost:8080/v1/mods/238222/files/4615177
+x-api-key: {{cfApiKey}}

dev/wiremock/data/mappings/.gitignore

@@ -0,0 +1 @@
+*.json

src/main/java/me/itzg/helpers/McImageHelper.java

@@ -12,6 +12,7 @@
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import me.itzg.helpers.assertcmd.AssertCommand;
+import me.itzg.helpers.curseforge.CurseForgeFilesCommand;
 import me.itzg.helpers.curseforge.InstallCurseForgeCommand;
 import me.itzg.helpers.errors.ExceptionHandler;
 import me.itzg.helpers.errors.ExitCodeMapper;
@@ -52,6 +53,7 @@
         Asciify.class,
         AssertCommand.class,
         CompareVersionsCommand.class,
+        CurseForgeFilesCommand.class,
         FindCommand.class,
         GetCommand.class,
         HashCommand.class,
@@ -87,8 +89,7 @@ public class McImageHelper {
   //language=RegExp
   public static final String VERSION_REGEX = "\\d+(\\.\\d+)+";
 
-  @SuppressWarnings("unused")
-  @CommandLine.Option(names = {"-h",
+  @Option(names = {"-h",
       "--help"}, usageHelp = true, description = "Show this usage and exit")
   boolean showHelp;
 
@@ -130,6 +131,7 @@ private static void setLevel(boolean enabled, Level level) {
   @Getter
   boolean silent;
 
+  @Getter
   private static String version;
 
   public static void main(String[] args) {
@@ -165,10 +167,6 @@ private static String loadVersion() throws IOException {
     return "???";
   }
 
-  public static String getVersion() {
-    return version;
-  }
-
   public static class AppVersionProvider implements IVersionProvider {
     @Override
     public String[] getVersion() {

src/main/java/me/itzg/helpers/curseforge/CategoryInfo.java

@@ -1,12 +1,31 @@
 package me.itzg.helpers.curseforge;
 
+import java.util.Map;
 import lombok.AllArgsConstructor;
 import me.itzg.helpers.curseforge.model.Category;
 
-import java.util.Map;
-
 @AllArgsConstructor
-class CategoryInfo {
+public class CategoryInfo {
     Map<Integer, Category> contentClassIds;
-    int modpackClassId;
+    Map<String, Integer> slugIds;
+
+    public int getClassIdForSlug(String categorySlug) {
+        final Integer classId = slugIds.get(categorySlug);
+        if (classId != null) {
+            return classId;
+        }
+        else {
+            throw new IllegalArgumentException("Unexpected category: " + categorySlug);
+        }
+    }
+
+    public Category getCategory(int categoryId) {
+        final Category category = contentClassIds.get(categoryId);
+        if (category != null) {
+            return category;
+        }
+        else {
+            throw new IllegalArgumentException("Unknown category ID: " + categoryId);
+        }
+    }
 }

src/main/java/me/itzg/helpers/curseforge/CurseForgeApiClient.java

@@ -6,9 +6,10 @@
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import lombok.extern.slf4j.Slf4j;
 import me.itzg.helpers.curseforge.model.Category;
 import me.itzg.helpers.curseforge.model.CurseForgeFile;
@@ -27,17 +28,27 @@
 import me.itzg.helpers.http.SharedFetch;
 import me.itzg.helpers.http.UriBuilder;
 import me.itzg.helpers.json.ObjectMappers;
+import org.slf4j.Logger;
 import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Schedulers;
 
 @Slf4j
 public class CurseForgeApiClient implements AutoCloseable {
+
+    public static final String CATEGORY_MODPACKS = "modpacks";
+    public static final String CATEGORY_MC_MODS = "mc-mods";
+    public static final String CATEGORY_BUKKIT_PLUGINS = "bukkit-plugins";
+    public static final String CATEGORY_WORLDS = "worlds";
+
     private static final String API_KEY_HEADER = "x-api-key";
+    static final String MINECRAFT_GAME_ID = "432";
 
     private final SharedFetch preparedFetch;
     private final UriBuilder uriBuilder;
     private final String gameId;
 
+    private final ConcurrentHashMap<Integer, CurseForgeMod> cachedMods = new ConcurrentHashMap<>();
+
     public CurseForgeApiClient(String apiBaseUrl, String apiKey, SharedFetch.Options sharedFetchOptions, String gameId
     ) {
         this.preparedFetch = Fetch.sharedFetch("install-curseforge",
@@ -48,57 +59,66 @@ public CurseForgeApiClient(String apiBaseUrl, String apiKey, SharedFetch.Options
         this.gameId = gameId;
     }
 
+    static FileDownloadStatusHandler modFileDownloadStatusHandler(Path outputDir, Logger log) {
+        return (status, uri, f) -> {
+            switch (status) {
+                case SKIP_FILE_EXISTS:
+                    log.info("Mod file {} already exists", outputDir.relativize(f));
+                    break;
+                case DOWNLOADED:
+                    log.info("Downloaded mod file {}", outputDir.relativize(f));
+                    break;
+            }
+        };
+    }
+
     @Override
     public void close() {
         preparedFetch.close();
     }
 
-    CategoryInfo loadModpacksCategoryInfo(Set<String> applicableClassIdSlugs) {
+    Mono<CategoryInfo> loadCategoryInfo(Collection<String> applicableClassIdSlugs) {
         return preparedFetch
             // get only categories that are classes, like mc-mods
-            .fetch(uriBuilder.resolve("/categories?gameId={gameId}&classesOnly=true", gameId))
+            .fetch(uriBuilder.resolve("/v1/categories?gameId={gameId}&classesOnly=true", gameId))
             .toObject(GetCategoriesResponse.class)
             .assemble()
             .flatMap(resp -> {
                     final Map<Integer, Category> contentClassIds = new HashMap<>();
-                    Integer modpackClassId = null;
+                    final Map<String, Integer> slugIds = new HashMap<>();
 
                     for (final Category category : resp.getData()) {
                         if (applicableClassIdSlugs.contains(category.getSlug())) {
                             contentClassIds.put(category.getId(), category);
-                        }
-                        if (category.getSlug().equals(CurseForgeInstaller.CATEGORY_SLUG_MODPACKS)) {
-                            modpackClassId = category.getId();
+                            slugIds.put(category.getSlug(), category.getId());
                         }
                     }
 
-                    if (modpackClassId == null) {
-                        return Mono.error(new GenericException("Unable to lookup classId for modpacks"));
-                    }
-
-                    return Mono.just(new CategoryInfo(contentClassIds, modpackClassId));
+                    return Mono.just(new CategoryInfo(contentClassIds, slugIds));
                 }
-            )
-            .block();
+            );
     }
 
-    CurseForgeMod searchMod(String slug, CategoryInfo categoryInfo) {
-        final ModsSearchResponse searchResponse = preparedFetch.fetch(
-                uriBuilder.resolve("/mods/search?gameId={gameId}&slug={slug}&classId={classId}",
-                    gameId, slug, categoryInfo.modpackClassId
+    Mono<CurseForgeMod> searchMod(String slug, int classId) {
+        return preparedFetch.fetch(
+                uriBuilder.resolve("/v1/mods/search?gameId={gameId}&slug={slug}&classId={classId}",
+                    gameId, slug, classId
                 )
             )
             .toObject(ModsSearchResponse.class)
-            .execute();
-
-        if (searchResponse.getData() == null || searchResponse.getData().isEmpty()) {
-            throw new GenericException("No mods found with slug={}" + slug);
-        } else if (searchResponse.getData().size() > 1) {
-            throw new GenericException("More than one mod found with slug=" + slug);
-        } else {
-            return searchResponse.getData().get(0);
-        }
-
+            .assemble()
+            .flatMap(searchResponse -> {
+                if (searchResponse.getData() == null || searchResponse.getData().isEmpty()) {
+                    return Mono.error(new GenericException("No mods found with slug=" + slug));
+                }
+                else if (searchResponse.getData().size() > 1) {
+                    return Mono.error(new GenericException("More than one mod found with slug=" + slug));
+                }
+                else {
+                    return Mono.just(searchResponse.getData().get(0));
+                }
+            })
+            .doOnNext(curseForgeMod -> cachedMods.put(curseForgeMod.getId(), curseForgeMod));
     }
 
     /**
@@ -110,7 +130,7 @@ public CurseForgeFile resolveModpackFile(
     ) {
         // NOTE latestFiles in mod is only one or two files, so retrieve the full list instead
         final GetModFilesResponse resp = preparedFetch.fetch(
-                uriBuilder.resolve("/mods/{modId}/files", mod.getId()
+                uriBuilder.resolve("/v1/mods/{modId}/files", mod.getId()
                 )
             )
             .toObject(GetModFilesResponse.class)
@@ -135,7 +155,7 @@ Mono<Integer> slugToId(CategoryInfo categoryInfo,
     ) {
         return preparedFetch
             .fetch(
-                uriBuilder.resolve("/mods/search?gameId={gameId}&slug={slug}", gameId, slug)
+                uriBuilder.resolve("/v1/mods/search?gameId={gameId}&slug={slug}", gameId, slug)
             )
             .toObject(ModsSearchResponse.class)
             .assemble()
@@ -153,13 +173,19 @@ public Mono<CurseForgeMod> getModInfo(
     ) {
         log.debug("Getting mod metadata for {}", projectID);
 
+        final CurseForgeMod cached = cachedMods.get(projectID);
+        if (cached != null) {
+            return Mono.just(cached);
+        }
+
         return preparedFetch.fetch(
-                uriBuilder.resolve("/mods/{modId}", projectID)
+                uriBuilder.resolve("/v1/mods/{modId}", projectID)
             )
             .toObject(GetModResponse.class)
             .assemble()
             .checkpoint("Getting mod info for " + projectID)
-            .map(GetModResponse::getData);
+            .map(GetModResponse::getData)
+            .doOnNext(curseForgeMod -> cachedMods.put(curseForgeMod.getId(), curseForgeMod));
     }
 
     public Mono<CurseForgeFile> getModFileInfo(
@@ -168,7 +194,7 @@ public Mono<CurseForgeFile> getModFileInfo(
         log.debug("Getting mod file metadata for {}:{}", projectID, fileID);
 
         return preparedFetch.fetch(
-                uriBuilder.resolve("/mods/{modId}/files/{fileId}", projectID, fileID)
+                uriBuilder.resolve("/v1/mods/{modId}/files/{fileId}", projectID, fileID)
             )
             .toObject(GetModFileResponse.class)
             .assemble()