H-3684: Upgrade to Yarn 4 (#5742)

Co-authored-by: Tim Diekmann <[email protected]>

.dockerignore

@@ -81,8 +81,14 @@ typings/
 # Output of 'npm pack'
 *.tgz
 
-# Yarn Integrity file
-.yarn-integrity
+# Yarn
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
 
 # local environment variable files
 blocks/**/.env

.github/actions/enable-corepack/action.yml

@@ -0,0 +1,9 @@
+name: Enable corepack
+description: "Enables corepack for the repository"
+
+runs:
+  using: composite
+  steps:
+    - name: Enable corepack
+      shell: bash
+      run: corepack enable

.github/actions/install-turbo/action.yml

@@ -6,4 +6,5 @@ runs:
   steps:
     - name: Install turbo
       shell: bash
-      run: yarn global add "turbo@$(jq -r '.devDependencies.turbo' < package.json)"
+      # `yarn global add` does not exist in yarn berry, instead you're supposed to use `yarn dlx` for oneshot scripts and `npm install -g` for global installs (if required)
+      run: npm install -g "turbo@$(jq -r '.devDependencies.turbo' < package.json)"

.github/actions/prune-repository/action.yml

@@ -8,17 +8,32 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - name: Enable corepack
+      uses: ./.github/actions/enable-corepack
+
     - name: Install turbo
       uses: ./.github/actions/install-turbo
 
     - name: Prune repository
       shell: bash
       run: |
-        while IFS= read -r line; do
-          if [[ -n "$line" ]]; then
-            turbo prune --scope="$line"
-          fi
-        done <<< "${{ inputs.scope }}"
+        # Convert newline-separated input to space-separated arguments
+        SCOPES=$(echo "${{ inputs.scope }}" | tr '\n' ' ')
+
+        # TODO: remove once deer is in labs
+        #   see https://linear.app/hash/issue/H-3692/move-deer-to-labs
+        # Nested crates are pruned in a way where the lockfile does not correctly reflect the pruned workspace, leading to an installation failure.
+        # Check if any of the scopes start with @rust/deer
+        # If any deer crate is in scope, include all deer crates
+        if echo "$SCOPES" | grep -q "@rust/deer\(-desert\|-macros\|-json\)\?"; then
+          SCOPES="$SCOPES @rust/deer @rust/deer-desert @rust/deer-macros @rust/deer-json"
+        fi
+
+        # Deduplicate scopes
+        SCOPES=$(echo "$SCOPES" | tr ' ' '\n' | sort -u | tr '\n' ' ')
+
+        # Run turbo prune with all scopes as arguments
+        turbo prune $SCOPES
 
         while IFS= read -r line; do
           DIRECTORY=$(dirname "$line")
@@ -31,24 +46,17 @@ runs:
             echo "name = \"$(yq '.package.name' -p toml -oy $line)\"" >> "out/$DIRECTORY/Cargo.toml"
             echo "edition.workspace = true" >> "out/$DIRECTORY/Cargo.toml"
           fi
-        done < <(find $(yq '.workspace.members' -p toml -o tsv Cargo.toml | tr '*' ' ') -maxdepth 2 -name Cargo.toml)
+        done < <(find $(yq '.workspace.members' -p toml -o tsv Cargo.toml | sed -e 's#/\*##g') -maxdepth 2 -name Cargo.toml | uniq)
 
     - name: Copy required files
       shell: bash
       run: |
-        cp -R yarn.lock patches Cargo.toml Cargo.lock rust-toolchain.toml out/
+        cp -R Cargo.toml Cargo.lock rust-toolchain.toml out/
 
         # Globs are fun, especially in Bash. Covers all dot-files except `.`, `..`, and `.git`.
         shopt -s extglob
         cp -R .[!.!(.git)]* .git?* out/
 
-    - name: Allow patch files to fail
-      shell: bash
-      run: |
-        # `patch-package` will always fail in CI if the package does not exist, `--error-on-fail` is not respected
-        # This allows `patch-package` to fail without failing the build
-        cat package.json | sed 's/\("postinstall".*patch-package.*\)"/\1 || true"/g' > out/package.json
-
     - name: Remove old repository files
       shell: bash
       run: |

.github/actions/warm-up-repo/action.yml

@@ -29,9 +29,13 @@ runs:
         max_attempts: 3
         timeout_minutes: 10
         shell: bash
+        # we disable hardened mode (https://yarnpkg.com/features/security), as it significantly slows down the installation
+        # immutable installs are still enforced in CI
         command: |
           export PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD="${{ inputs.playwright-deps == '' }}"
-          yarn install --frozen-lockfile --prefer-offline
+          export YARN_ENABLE_HARDENED_MODE=0
+          corepack enable
+          yarn install --immutable
 
     - name: npx playwright install
       if: ${{ inputs.playwright-deps != '' }}

.github/labeler.yml

@@ -42,7 +42,7 @@ area/blocks:
 
 area/deps:
   - changed-files:
-      - any-glob-to-any-file: "**/patches/**"
+      - any-glob-to-any-file: "**/.yarn/**"
       - any-glob-to-any-file: "**/Cargo.lock"
       - any-glob-to-any-file: "**/yarn.lock"
   - head-branch: ["^deps", "deps"]

.github/workflows/bench.yml

@@ -29,6 +29,9 @@ jobs:
         with:
           fetch-depth: 2
 
+      - name: Enable corepack
+        uses: ./.github/actions/enable-corepack
+
       - name: Install turbo
         uses: ./.github/actions/install-turbo
 
@@ -68,6 +71,9 @@ jobs:
         with:
           fetch-depth: 2
 
+      - name: Enable corepack
+        uses: ./.github/actions/enable-corepack
+
       - name: Install turbo
         uses: ./.github/actions/install-turbo
 
@@ -217,6 +223,9 @@ jobs:
         with:
           fetch-depth: 2
 
+      - name: Enable corepack
+        uses: ./.github/actions/enable-corepack
+
       - name: Install turbo
         uses: ./.github/actions/install-turbo
 

.github/workflows/deploy.yml

@@ -27,6 +27,9 @@ jobs:
         with:
           fetch-depth: 2
 
+      - name: Enable corepack
+        uses: ./.github/actions/enable-corepack
+
       - name: Install turbo
         uses: ./.github/actions/install-turbo
 
@@ -69,6 +72,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
+      - name: Enable corepack
+        uses: ./.github/actions/enable-corepack
+
       - name: Install turbo
         uses: ./.github/actions/install-turbo
 

.github/workflows/lint.yml

@@ -27,6 +27,9 @@ jobs:
         with:
           fetch-depth: 2
 
+      - name: Enable corepack
+        uses: ./.github/actions/enable-corepack
+
       - name: Install turbo
         uses: ./.github/actions/install-turbo
 
@@ -55,6 +58,9 @@ jobs:
         with:
           fetch-depth: 2
 
+      - name: Enable corepack
+        uses: ./.github/actions/enable-corepack
+
       - name: Install turbo
         uses: ./.github/actions/install-turbo
 

.github/workflows/test.yml

@@ -31,6 +31,9 @@ jobs:
         with:
           fetch-depth: 2
 
+      - name: Enable corepack
+        uses: ./.github/actions/enable-corepack
+
       - name: Install turbo
         uses: ./.github/actions/install-turbo
 
@@ -109,6 +112,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
+      - name: Enable corepack
+        uses: ./.github/actions/enable-corepack
+
       - name: Install turbo
         uses: ./.github/actions/install-turbo
 
@@ -230,6 +236,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
+      - name: Enable corepack
+        uses: ./.github/actions/enable-corepack
+
       - name: Install turbo
         uses: ./.github/actions/install-turbo
 

.gitignore

@@ -65,8 +65,14 @@ typings/
 # Output of 'npm pack'
 *.tgz
 
-# Yarn Integrity file
-.yarn-integrity
+# Yarn
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
 
 # local environment variable files
 blocks/**/.env

.markdownlintignore

@@ -74,8 +74,14 @@ typings/
 # Output of 'npm pack'
 *.tgz
 
-# Yarn Integrity file
-.yarn-integrity
+# Yarn
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
 
 # local environment variable files
 blocks/**/.env

.prettierignore

@@ -91,8 +91,14 @@ typings/
 # Output of 'npm pack'
 *.tgz
 
-# Yarn Integrity file
-.yarn-integrity
+# Yarn
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
 
 # local environment variable files
 blocks/**/.env

patches/@artilleryio+int-commons+2.11.0.patch → .yarn/patches/@artilleryio-int-commons-npm-2.11.0-5b69c05121.patch

@@ -1,7 +1,7 @@
-diff --git a/node_modules/@artilleryio/int-commons/engine_util.js b/node_modules/@artilleryio/int-commons/engine_util.js
-index 8e5dc73..31a9a01 100644
---- a/node_modules/@artilleryio/int-commons/engine_util.js
-+++ b/node_modules/@artilleryio/int-commons/engine_util.js
+diff --git a/engine_util.js b/engine_util.js
+index 8e5dc732ada44f9b17e177bf8cb684af5f55ae37..31a9a0100491fa153ff00eeac89ce74695eb9a12 100644
+--- a/engine_util.js
++++ b/engine_util.js
 @@ -6,7 +6,6 @@
 
  const async = require('async');

patches/@changesets+assemble-release-plan+5.2.3.patch → .yarn/patches/@changesets-assemble-release-plan-npm-5.2.4-2920e4dc4c.patch

@@ -1,16 +1,16 @@
-diff --git a/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.cjs.dev.js b/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.cjs.dev.js
-index 3a37c62..146db90 100644
---- a/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.cjs.dev.js
-+++ b/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.cjs.dev.js
-@@ -145,6 +145,7 @@ function determineDependents({
-           } else if ((!releases.has(dependent) || releases.get(dependent).type === "none") && (config.___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH.updateInternalDependents === "always" || !semver__default['default'].satisfies(incrementVersion(nextRelease, preInfo), versionRange))) {
+diff --git a/dist/assemble-release-plan.cjs.dev.js b/dist/assemble-release-plan.cjs.dev.js
+index e1376ca756d69816f8c79637ee7b45161f092167..7825c0cfbdf53e5070c21ff5dcf74242a7d4be41 100644
+--- a/dist/assemble-release-plan.cjs.dev.js
++++ b/dist/assemble-release-plan.cjs.dev.js
+@@ -151,6 +151,7 @@ function determineDependents({
+           } else if ((!releases.has(dependent) || releases.get(dependent).type === "none") && (config.___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH.updateInternalDependents === "always" || !semverSatisfies__default['default'](incrementVersion(nextRelease, preInfo), versionRange))) {
              switch (depType) {
                case "dependencies":
 +              case "devDependencies":
                case "optionalDependencies":
                case "peerDependencies":
                  if (type !== "major" && type !== "minor") {
-@@ -152,14 +153,6 @@ function determineDependents({
+@@ -158,14 +159,6 @@ function determineDependents({
                  }
 
                  break;
@@ -25,13 +25,13 @@ index 3a37c62..146db90 100644
              }
            }
          }
-diff --git a/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.cjs.prod.js b/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.cjs.prod.js
-index 87b4c10..ad83a16 100644
---- a/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.cjs.prod.js
-+++ b/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.cjs.prod.js
+diff --git a/dist/assemble-release-plan.cjs.prod.js b/dist/assemble-release-plan.cjs.prod.js
+index 3a83720644a94cdf6e62fa188a72c51c0384d00e..4d41e9627b05bc7a71ed886b4687467c4a3cbb82 100644
+--- a/dist/assemble-release-plan.cjs.prod.js
++++ b/dist/assemble-release-plan.cjs.prod.js
 @@ -80,13 +80,11 @@ function determineDependents({releases: releases, packagesByName: packagesByName
            onlyUpdatePeerDependentsWhenOutOfRange: config.___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH.onlyUpdatePeerDependentsWhenOutOfRange
-         })) type = "major"; else if (!(releases.has(dependent) && "none" !== releases.get(dependent).type || "always" !== config.___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH.updateInternalDependents && semver__default.default.satisfies(incrementVersion(nextRelease, preInfo), versionRange))) switch (depType) {
+         })) type = "major"; else if (!(releases.has(dependent) && "none" !== releases.get(dependent).type || "always" !== config.___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH.updateInternalDependents && semverSatisfies__default.default(incrementVersion(nextRelease, preInfo), versionRange))) switch (depType) {
           case "dependencies":
 +         case "devDependencies":
           case "optionalDependencies":
@@ -44,19 +44,19 @@ index 87b4c10..ad83a16 100644
          }
        }
        return releases.has(dependent) && releases.get(dependent).type === type && (type = void 0), 
-diff --git a/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.esm.js b/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.esm.js
-index c29c008..5632dfb 100644
---- a/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.esm.js
-+++ b/node_modules/@changesets/assemble-release-plan/dist/assemble-release-plan.esm.js
-@@ -137,6 +137,7 @@ function determineDependents({
-           } else if ((!releases.has(dependent) || releases.get(dependent).type === "none") && (config.___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH.updateInternalDependents === "always" || !semver.satisfies(incrementVersion(nextRelease, preInfo), versionRange))) {
+diff --git a/dist/assemble-release-plan.esm.js b/dist/assemble-release-plan.esm.js
+index 62891eb5dee97a33e6587514267c3cde5b314830..f0d058a69df3375f08aa0cbbdaf0897bb9fc20e7 100644
+--- a/dist/assemble-release-plan.esm.js
++++ b/dist/assemble-release-plan.esm.js
+@@ -140,6 +140,7 @@ function determineDependents({
+           } else if ((!releases.has(dependent) || releases.get(dependent).type === "none") && (config.___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH.updateInternalDependents === "always" || !semverSatisfies(incrementVersion(nextRelease, preInfo), versionRange))) {
              switch (depType) {
                case "dependencies":
 +              case "devDependencies":
                case "optionalDependencies":
                case "peerDependencies":
                  if (type !== "major" && type !== "minor") {
-@@ -144,14 +145,6 @@ function determineDependents({
+@@ -147,14 +148,6 @@ function determineDependents({
                  }
 
                  break;

patches/@changesets+cli+2.26.0.patch → .yarn/patches/@changesets-cli-npm-2.26.0-49d5c5f72d.patch

@@ -1,7 +1,7 @@
-diff --git a/node_modules/@changesets/cli/dist/cli.cjs.dev.js b/node_modules/@changesets/cli/dist/cli.cjs.dev.js
-index b158219..0da127e 100644
---- a/node_modules/@changesets/cli/dist/cli.cjs.dev.js
-+++ b/node_modules/@changesets/cli/dist/cli.cjs.dev.js
+diff --git a/dist/cli.cjs.dev.js b/dist/cli.cjs.dev.js
+index b1582198d3d2631569ff59f27170354a932d3ad0..0da127e040352b530af89a1ce2d8b29ab6bec9c1 100644
+--- a/dist/cli.cjs.dev.js
++++ b/dist/cli.cjs.dev.js
 @@ -1176,7 +1176,8 @@ async function getStatus(cwd, {
    if (changedPackages.length > 0 && changesets.length === 0) {
      logger.error("Some packages have been changed but no changesets were found. Run `changeset add` to resolve this error.");
@@ -12,10 +12,10 @@ index b158219..0da127e 100644
    }
 
    if (output) {
-diff --git a/node_modules/@changesets/cli/dist/cli.cjs.prod.js b/node_modules/@changesets/cli/dist/cli.cjs.prod.js
-index 5b1b7dd..94ea167 100644
---- a/node_modules/@changesets/cli/dist/cli.cjs.prod.js
-+++ b/node_modules/@changesets/cli/dist/cli.cjs.prod.js
+diff --git a/dist/cli.cjs.prod.js b/dist/cli.cjs.prod.js
+index 5b1b7dd6439be4f0b721a17e683b1435c254f5b8..94ea1672f1dc5ec661b0e71d285d1b68d69d48d1 100644
+--- a/dist/cli.cjs.prod.js
++++ b/dist/cli.cjs.prod.js
 @@ -635,7 +635,8 @@ async function getStatus(cwd, {sinceMaster: sinceMaster, since: since, verbose:
      changedFilePatterns: config.changedFilePatterns
    })).length > 0 && 0 === changesets.length && (logger.error("Some packages have been changed but no changesets were found. Run `changeset add` to resolve this error."), 
@@ -26,10 +26,10 @@ index 5b1b7dd..94ea167 100644
    const print = verbose ? verbosePrint : SimplePrint;
    return print("patch", releases), logger.log("---"), print("minor", releases), logger.log("---"), 
    print("major", releases), releasePlan;
-diff --git a/node_modules/@changesets/cli/dist/cli.esm.js b/node_modules/@changesets/cli/dist/cli.esm.js
-index ced46d2..cf14334 100644
---- a/node_modules/@changesets/cli/dist/cli.esm.js
-+++ b/node_modules/@changesets/cli/dist/cli.esm.js
+diff --git a/dist/cli.esm.js b/dist/cli.esm.js
+index ced46d2520d4bff71b94e4023b1841373a55712b..cf14334e20f96b52c5b63f4c61933ab25c4b7de8 100644
+--- a/dist/cli.esm.js
++++ b/dist/cli.esm.js
 @@ -1153,7 +1153,8 @@ async function getStatus(cwd, {
    if (changedPackages.length > 0 && changesets.length === 0) {
      error("Some packages have been changed but no changesets were found. Run `changeset add` to resolve this error.");