Feature: RVZSTD support, candidate selection improvements (#1563)

Author: emmercm

.github/CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Contribution Guidelines
 
-Welcome! If you're viewing this, it means that you’re likely interested in contributing to the project. That's fantastic!
+Welcome! If you're viewing this, it means that you're likely interested in contributing to the project. That's fantastic!
 
 The following is a set of guidelines for contributing to Igir. These guidelines are published to clarify expectations and reduce potential conflict of opinions.
 

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -43,7 +43,7 @@ body:
       description: |
         The debug logs of your command run with the `-vvv` option. This is _not_ just a copy of the standard progress bar CLI output.
 
-        You can redirect the CLI output of `igir` to a file with the syntax: `igir [commands..] [options] -vvv > igir.log`. You can then attach this file in the GitHub text editor.
+        You can redirect the CLI output of Igir to a file with the syntax: `igir [commands..] [options] -vvv > igir.log`. You can then attach this file in the GitHub text editor.
 
         This field is _not_ optional as it is one of the most important ways to gain insight into your unique scenario. Without this information it is difficult to reproduce your problem.
     validations:

.github/renovate.json5

@@ -126,12 +126,6 @@
         },
       },
 
-      {
-        // Locked to a specific Zstandard version for RVZSTD zip creation
-        matchPackageNames: ['zstd-napi'],
-        enabled: false,
-      },
-
       // Update chdman, dolphin-tool, maxcso immediately
       {
         groupName: null,

.github/workflows/codecov.yml

@@ -37,7 +37,7 @@ jobs:
         with:
           submodules: 'recursive'
       - uses: volta-cli/action@v4
-      - run: npm ci
+      - run: npm ci --foreground-scripts
       - run: |
           sudo apt-get update
           sudo apt-get install -y libsdl2-2.0-0 libsdl2-ttf-2.0-0

.github/workflows/napi-prebuild.yml

@@ -1,12 +1,9 @@
 name: Node-API Prebuild
 
 on:
-  pull_request:
-    types:
-      # - edited  # PR's base branch was changed
-      - opened
-      - reopened
-      - synchronize  # PR's branch was edited (i.e. new commits)
+  push:
+    branches-ignore:
+      - 'main'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -30,14 +27,19 @@ jobs:
           base: ${{ github.ref }}
           filters: |
             changes:
-              - 'packages/torrentzip/*'
+              - '.github/workflows/napi-prebuild.yml'
+              - 'packages/zlib*/**'
+              - 'packages/zstd*/**'
 
   prebuild:
     needs:
       - path-filter
     if: ${{ needs.path-filter.outputs.changes == 'true' }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 10
+    defaults:
+      run:
+        shell: bash
+    timeout-minutes: 30 # QEMU cross-builds
     name: prebuildify (${{ matrix.os }} ${{ matrix.docker_arch }})
     strategy:
       fail-fast: false
@@ -48,10 +50,10 @@ jobs:
             docker_arch: linux/amd64
           - os: ubuntu-latest
             docker_arch: linux/arm/v7
-          - os: ubuntu-latest
+          - os: ubuntu-24.04-arm
             docker_arch: linux/arm64/v8
           - os: macos-13
-          - os: macos-15
+          - os: macos-latest
           - os: windows-latest
           - os: windows-11-arm
     steps:
@@ -60,48 +62,70 @@ jobs:
         with:
           submodules: 'recursive'
       - uses: volta-cli/action@v4
+        with:
+          node-version: 18
 
       # Prebuild
       - shell: bash
         run: |
+          set -x
+          npm ci --ignore-scripts
           rm -rf packages/*/prebuilds
       - if: ${{ startsWith(matrix.os, 'ubuntu') }}
         uses: docker/setup-qemu-action@v3
       - id: linux-vars
         if: ${{ startsWith(matrix.os, 'ubuntu') }}
         run: |
-          echo "VOLTA_NODE=$(jq --raw-output '.volta.node' package.json)" >> "${GITHUB_OUTPUT}"
+          echo "VERSIONS_NODE=$(node --print 'process.versions.node')" >> "${GITHUB_OUTPUT}"
 
           DOCKER_ARCH=${{ matrix.docker_arch }}
           DOCKER_ARCH=${DOCKER_ARCH/linux\//}
           DOCKER_ARCH=${DOCKER_ARCH//\//}
           echo "DOCKER_ARCH=${DOCKER_ARCH}" >> "${GITHUB_OUTPUT}"
       - if: ${{ startsWith(matrix.os, 'ubuntu') }}
+        id: build-docker
         uses: addnab/docker-run-action@v3
         with:
-          image: node:${{ steps.linux-vars.outputs.VOLTA_NODE }}
+          image: node:${{ steps.linux-vars.outputs.VERSIONS_NODE }}
           shell: bash
           options: |
             --platform ${{ matrix.docker_arch }}
             --volume ${{ github.workspace }}:/build
             --workdir /build
           run: |
             set -x
-            cd packages/torrentzip
-            npm install --ignore-scripts
-            ./node_modules/.bin/prebuildify --napi --strip
-            touch prebuilds/.gitkeep
+            find packages -maxdepth 1 -type d \( -name "zlib*" -o -name "zstd*" \) | while IFS= read -r dir; do
+              cd "${dir}"
+              npm install --ignore-scripts
+              ./node_modules/.bin/prebuildify --target "$(node --print 'process.versions.node')" --napi --strip
+              touch prebuilds/.gitkeep
+              cd ../..
+            done
+            ls -alR packages/*/prebuilds/
+
+            # Test the build
+            rm -rf packages/*/build
+            npm ci --foreground-scripts
+            npm run test:unit packages
       - if: ${{ !startsWith(matrix.os, 'ubuntu') }}
-        shell: bash
+        id: build
         run: |
           set -x
-          cd packages/torrentzip
-          npm install --ignore-scripts
-          ./node_modules/.bin/prebuildify --napi --strip
-          touch prebuilds/.gitkeep
+          find packages -maxdepth 1 -type d \( -name "zlib*" -o -name "zstd*" \) | while IFS= read -r dir; do
+            cd "${dir}"
+            npm install --ignore-scripts
+            ./node_modules/.bin/prebuildify --target "$(node --print 'process.versions.node')" --napi --strip
+            touch prebuilds/.gitkeep
+            cd ../..
+          done
+          ls -alR packages/*/prebuilds/
 
-      - shell: bash
-        run: |
+          # Test the build
+          rm -rf packages/*/build
+          npm ci --foreground-scripts
+          npm run test:unit packages
+
+      - run: |
           ls -alR packages/*/prebuilds/
       - uses: actions/upload-artifact@v4
         with:

.github/workflows/node-compile.yml

@@ -63,7 +63,7 @@ jobs:
             docker_arch: linux/arm64/v8
 
           - os: macos-13
-          - os: macos-15
+          - os: macos-latest
 
           - os: windows-latest
             go_arch: amd64
@@ -118,6 +118,8 @@ jobs:
               ls -alh igir "${{ env.BIN_OUTPUT }}"
               echo '```'
             } >> "${GITHUB_STEP_SUMMARY:-/dev/null}"
+      - if: ${{ startsWith(matrix.os, 'ubuntu') }}
+        run: sudo chown -R "$(id -u):$(id -g)" .
 
       - if: ${{ startsWith(matrix.os, 'macos') }}
         run: |
@@ -148,18 +150,7 @@ jobs:
           echo "BIN_OUTPUT=${BIN_OUTPUT}" | Out-File -FilePath $env:GITHUB_ENV -Append # no need for -Encoding utf8
 
           # Install dependencies...
-          if ($NODE_ARCH -in @("x64")) {
-            npm ci
-          } else {
-            # ...but fix zstd-napi not compiling Zstandard correctly out of the box
-            npm ci --ignore-scripts
-            Push-Location
-            cd node_modules\zstd-napi
-            Set-Content -Path deps\zstd.gyp -Value (Get-Content -Path deps\zstd.gyp | Select-String -Pattern 'huf_decompress_amd64' -NotMatch)
-            npm install node-gyp
-            node_modules\.bin\node-gyp rebuild
-            Pop-Location
-          }
+          npm ci
           if ($NODE_ARCH -notin @("ia32", "x64")) {
             # ...and use a version of 7zip-bin that includes this arch
             npm install 7zip-bin@">=5.2.0"

.github/workflows/node-test.yml

@@ -75,8 +75,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-24.04, ubuntu-24.04-arm, macos-15, windows-latest ]
-        node-version: [ lts, 18 ]
+        # TODO(cemmer): chdman-js and maxcso-js don't have Windows ARM builds, so it won't test the same
+        os: [ ubuntu-latest, ubuntu-24.04-arm, macos-latest, macos-13, windows-latest ]
+        node-version: [ lts ]
+        include:
+          - os: ubuntu-latest
+            node-version: 18
+          - os: macos-latest
+            node-version: 18
+          - os: windows-latest
+            node-version: 18
     steps:
       # Setup and install
       - uses: actions/checkout@v4
@@ -86,7 +94,10 @@ jobs:
       - uses: volta-cli/action@v4
         with:
           node-version: ${{ matrix.node-version }}
-      - run: npm ci
+      - shell: bash
+        run: |
+          rm -rf packages/*/prebuilds
+          npm ci --foreground-scripts
       - if: ${{ startsWith(matrix.os, 'ubuntu') }}
         run: |
           sudo apt-get update
@@ -103,7 +114,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        node-version: [ lts, 18, ]
+        node-version: [ lts, 18 ]
     steps:
       # Setup and install
       - uses: actions/checkout@v4
@@ -113,7 +124,7 @@ jobs:
       - uses: volta-cli/action@v4
         with:
           node-version: ${{ matrix.node-version }}
-      - run: npm ci
+      - run: npm ci --foreground-scripts
       - run: |
           sudo apt-get update
           sudo apt-get install -y libsdl2-2.0-0 libsdl2-ttf-2.0-0
@@ -144,7 +155,7 @@ jobs:
           npm pack
           tar -xvzf igir-*.tgz
           cd package
-          npm install
+          npm install --foreground-scripts
           ./dist/index.js --help
           cd ..
           rm -rf package

.gitignore

@@ -113,6 +113,8 @@ site/
 *.bat
 *.cache*
 *.sh
+igir
+igir.exe
 !test/*.sh
 
 # DATs

.gitmodules

@@ -1,6 +1,6 @@
-[submodule "packages/torrentzip/deps/zlib"]
-	path = packages/torrentzip/deps/zlib
-	url = https://github.com/madler/zlib.git
-[submodule "packages/torrentzip/deps/zlib_1_1_3"]
-	path = packages/torrentzip/deps/zlib_1_1_3
+[submodule "packages/zlib-1.1.3/deps/zlib"]
+	path = packages/zlib-1.1.3/deps/zlib
 	url = https://github.com/madler/zlib.git
+[submodule "packages/zstd-1.5.5/deps/zstd"]
+	path = packages/zstd-1.5.5/deps/zstd
+	url = https://github.com/facebook/zstd.git