tests.yml

name: Tests

on:
  pull_request:
  push:
    branches: [ 'trunk', '*/branch-*' ]

concurrency:
  # Trunk runs need to not be cancelled for concurrency, mainly for code coverage. Everything else can be.
  group: tests-${{ github.event_name }}-${{ github.ref }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' && github.run_id || '' }}
  cancel-in-progress: true

env:
  COMPOSER_ROOT_VERSION: "dev-trunk"

jobs:
  create-matrix:
    name: "Determine tests matrix"
    runs-on: ubuntu-latest
    timeout-minutes: 1  # 2021-02-03: Should only take a second.
    outputs:
      matrix: ${{ steps.create-matrix.outputs.matrix }}
    steps:
      - uses: actions/checkout@v4
      - id: create-matrix
        run: |
          MATRIX="$(.github/files/generate-ci-matrix.php)"
          echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"

  run-tests:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.runner }}
    needs: create-matrix
    services:
      database:
        image: mariadb:10.4
        env:
          MARIADB_ROOT_PASSWORD: root
        ports:
          - 3306:3306
        options: --health-cmd="healthcheck.sh --su-mysql --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=5
    continue-on-error: ${{ matrix.experimental }}
    timeout-minutes: ${{ matrix.timeout }}
    env:
      TEST_SCRIPT: ${{ matrix.script }}
      WP_BRANCH: ${{ matrix.wp }}
      PHP_VERSION: ${{ matrix.php }}
      NODE_VERSION: ${{ matrix.node }}
      MONOREPO_BASE: ${{ github.workspace }}
      WITH_WOOCOMMERCE: ${{ matrix.with-woocommerce }}
      WITH_WPCOMSH: ${{ matrix.with-wpcomsh }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson( needs.create-matrix.outputs.matrix ) }}

    # Note matrix-job outputs are kind of weird. Last-to-run job that sets a non-empty value wins.
    outputs:
      did-coverage: ${{ ( steps.run-tests.conclusion != 'cancelled' && steps.process-coverage.conclusion == 'success' && steps.upload-artifacts.conclusion == 'success' ) && 'true' || '' }}
      coverage-status: ${{ matrix.script == 'test-coverage' && steps.run-tests.conclusion || '' }}

    steps:
      - uses: actions/checkout@v4
        with:
          # Test coverage checks require a fetch depth > 1.
          fetch-depth: 2

      # For pull requests, list-changed-projects.sh needs the merge base.
      # But it doesn't have to be checked out.
      - name: Deepen to merge base
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/deepen-to-merge-base
        with:
          checkout: false

      - name: Setup tools
        uses: ./.github/actions/tool-setup
        with:
          php: ${{ matrix.php }}
          coverage: ${{ matrix.script == 'test-coverage' && 'pcov' || 'none' }}
          node: ${{ matrix.node }}

      - name: Monorepo install
        run: |
          echo "::group::Pnpm"
          pnpm install
          echo "::endgroup::"

      - name: Detect changed projects
        id: changed
        run: |
          CHANGED="$(EXTRA=test .github/files/list-changed-projects.sh)"

          # Only test certain plugins in combination with WC
          if [[ "$WITH_WOOCOMMERCE" == true ]]; then
            echo "Testing with WooCommerce, filtering for projects that have WooCommerce-specific tests."
            CHANGED=$( jq -c 'with_entries( select( .key == "plugins/jetpack" ) )' <<<"$CHANGED" )
          fi

          ANY_PLUGINS="$(jq --argjson changed "$CHANGED" -n '$changed | with_entries( select( .key | startswith( "plugins/" ) ) ) | any')"
          echo "projects=${CHANGED}" >> "$GITHUB_OUTPUT"
          echo "any-plugins=${ANY_PLUGINS}" >> "$GITHUB_OUTPUT"

      - name: Select WordPress version
        if: matrix.wp != 'none'
        run: .github/files/select-wordpress-tag.sh

      - name: Composer Install
        run: |
          # If we're going to be making WorDBless use WP "nightlies", remove the relevant package from Composer's cache to get the latest version.
          if [[ "$WP_BRANCH" == 'trunk' && ( "$TEST_SCRIPT" == "test-php" || "$TEST_SCRIPT" == "test-coverage" ) ]]; then
            echo "::group::Clear composer cache for roots/wordpress"
            DIR=$(composer config cache-files-dir)
            rm -rf "$DIR/roots/wordpress" "$DIR/roots/wordpress-no-content"
            echo "::endgroup::"
          fi

          echo "::group::Composer"
          composer install --working-dir=tools/php-test-env

          # Add WorDBless for various package tests that use uploads since concurrency can nuke uploads that are in use.
          # @todo Update WorDBless to be able to use a customized upload dir, do that, and remove this.
          if [[ "$TEST_SCRIPT" =~ ^test-(php|coverage)$ ]]; then
            VER=$( jq -r '.require["automattic/wordbless"]' tools/php-test-env/composer.json )
            for pkg in packages/image-cdn packages/publicize packages/videopress; do
              echo "Adding WordBless for $pkg tests"
              composer require --working-dir="projects/$pkg" "automattic/wordbless:$VER" --dev --no-update
            done
          fi

          if [[ ( "$TEST_SCRIPT" == "test-php" || "$TEST_SCRIPT" == "test-coverage" ) && ( "$WP_BRANCH" == 'trunk' || "$WP_BRANCH" == 'previous' ) ]]; then
            VER=$(composer --format=json --working-dir="tools/php-test-env" show | jq -r '.installed[] | select( .name == "roots/wordpress" ) | .version')
            if [[ -n "$VER" ]]; then
              INSVER=$WORDPRESS_TAG
              [[ "$WORDPRESS_TAG" == 'trunk' ]] && INSVER="dev-main as $VER"
              echo "Supposed to run tests against WordPress $WORDPRESS_TAG, so setting roots/wordpress and roots/wordpress-no-content to \"$INSVER\""
              # Composer seems to sometimes have issues with deleting the wordpress dir on its own, so do it manually first.
              rm -rf "tools/php-test-env/wordpress"
              composer --working-dir="tools/php-test-env" require --dev "roots/wordpress:$INSVER" "roots/wordpress-no-content:$INSVER"
            fi
          fi
          echo "::endgroup::"

      - name: Setup WordPress environment for plugin tests
        env:
          API_TOKEN_GITHUB: ${{ secrets.GITHUB_TOKEN }}
          CHANGED: ${{ steps.changed.outputs.projects }}
        if: steps.changed.outputs.any-plugins == 'true' && matrix.wp != 'none'
        run: .github/files/setup-wordpress-env.sh

      - name: Run project tests
        id: run-tests
        env:
          FORCE_PACKAGE_TESTS: ${{ matrix.force-package-tests && 'true' || 'false' }}
          CHANGED: ${{ steps.changed.outputs.projects }}
        run: |
          EXIT=0
          declare -A PIDS
          PIDS=()
          MAXPIDS=$( nproc )
          FAILED=()
          mkdir artifacts
          [[ "$TEST_SCRIPT" == "test-coverage" ]] && mkdir coverage
          for P in composer.json projects/*/*/composer.json; do
            if [[ ${#PIDS[@]} -ge $MAXPIDS ]]; then
              if ! wait -fn -p PID "${!PIDS[@]}"; then
                echo "::error::Tests for ${PIDS[$PID]} failed!"
                FAILED+=( "${PIDS[$PID]}" )
                EXIT=1
              fi
              echo "Finished ${PIDS[$PID]}"
              unset PIDS[$PID]
            fi

            if [[ "$P" == "composer.json" ]]; then
              DIR="."
              SLUG="monorepo"
            else
              DIR="${P%/composer.json}"
              SLUG="${DIR#projects/}"
            fi

            if [[ "${SLUG%%/*}" != "plugins" && "$WP_BRANCH" != 'latest' && "$WP_BRANCH" != 'none' && "$FORCE_PACKAGE_TESTS" != "true" ]]; then
              echo "Skipping $SLUG, only plugins run for WP_BRANCH = $WP_BRANCH"
              continue
            fi

            if ! jq --argjson changed "$CHANGED" --arg p "$SLUG" -ne '$changed[$p] // false' > /dev/null; then
              echo "Skipping $SLUG, no changes in it or its dependencies"
            elif ! jq --arg script "$TEST_SCRIPT" -e '.scripts[$script] // false' "$P" > /dev/null; then
              echo "Skipping $SLUG, no test script is defined in composer.json"
            elif php -r 'exit( preg_match( "/^>=\\s*(\\d+\\.\\d+)$/", $argv[1], $m ) && version_compare( PHP_VERSION, $m[1], "<" ) ? 0 : 1 );' "$( jq -r '.require.php // ""' "$P" )"; then
              echo "Skipping $SLUG, requires PHP $( jq -r '.require.php // ""' "$P" ) but PHP version is $( php -r 'echo PHP_VERSION;' )"
            else
              if jq --arg script "skip-$TEST_SCRIPT" -e '.scripts[$script] // false' "$P" > /dev/null; then
                { composer --working-dir="$DIR" run "skip-$TEST_SCRIPT"; CODE=$?; } || true
                if [[ $CODE -eq 3 ]]; then
                  echo "Skipping tests for $SLUG due to skip-$TEST_SCRIPT script"
                  continue
                elif [[ $CODE -ne 0 ]]; then
                  echo "::error::Script skip-$TEST_SCRIPT failed to run $CODE!"
                  FAILED+=( "$SLUG" )
                  EXIT=1
                  continue
                fi
              fi

              echo "Running tests for $SLUG"
              {
                # Composer install, if appropriate. Note setup-wordpress-env.sh did it already for plugins.
                if [[ "${SLUG%%/*}" != "plugins" && ( "$TEST_SCRIPT" == "test-php" || "$TEST_SCRIPT" == "test-coverage" ) ]]; then
                  if [[ "$TEST_SCRIPT" == "test-coverage" ]] &&
                    ! jq -e '.scripts["test-php"]' "$DIR/composer.json" &>/dev/null
                  then
                    echo "Skipping composer install, assuming test-coverage is only JS because the project has no test-php."
                  else
                    if [[ ! -f "$DIR/composer.lock" ]]; then
                      echo 'No composer.lock, running `composer update`'
                      composer --working-dir="$DIR" update
                    elif composer --working-dir="$DIR" check-platform-reqs --lock; then
                      echo 'Platform reqs pass, running `composer install`'
                      composer --working-dir="$DIR" install
                    else
                      echo 'Platform reqs failed, running `composer update`'
                      composer --working-dir="$DIR" update
                    fi
                  fi
                fi

                if [[ "${SLUG%%/*}" == "plugins" ]]; then
                  export WP_TESTS_CONFIG_FILE_PATH="$WORDPRESS_DEVELOP_DIR/wp-tests-config.${SLUG##*/}.php"
                fi

                mkdir -p "artifacts/$SLUG"
                export ARTIFACTS_DIR="$GITHUB_WORKSPACE/artifacts/$SLUG"
                if [[ "$TEST_SCRIPT" == "test-coverage" ]]; then
                  mkdir -p "coverage/$SLUG"
                  export COVERAGE_DIR="$GITHUB_WORKSPACE/coverage/$SLUG"
                fi
                FAIL=false
                if ! composer run --timeout=0 --working-dir="$DIR" "$TEST_SCRIPT"; then
                  FAIL=true
                fi

                # Actions seems to slow down if there are a lot of files, so clean up Composer stuff after each test.
                # We don't do it for JS stuff, as that might break things with how JS does package deps.
                rm -rf "$DIR/vendor" "$DIR/jetpack_vendor" "$DIR/wordpress"

                if $FAIL; then
                  echo "Tests for $SLUG failed!"
                  exit 1
                fi
              } 2> >( sed -u 's!^!['"$SLUG"'] !' >&2 ) > >( sed -u 's!^!['"$SLUG"'] !' ) &
              PIDS[$!]=$SLUG
            fi
          done
          while [[ ${#PIDS[@]} -gt 0 ]]; do
            if ! wait -fn -p PID "${!PIDS[@]}"; then
              echo "::error::Tests for ${PIDS[$PID]} failed!"
              FAILED+=( "${PIDS[$PID]}" )
              EXIT=1
            fi
            echo "Finished ${PIDS[$PID]}"
            unset PIDS[$PID]
          done

          if [[ ${#FAILED[@]} -gt 0 ]]; then
            echo ''
            echo 'The following tests failed:'
            printf " - %s\n" "${FAILED[@]}"
          fi

          exit $EXIT

      - name: Process coverage results
        id: process-coverage
        env:
          CHANGED: ${{ steps.changed.outputs.projects }}
        if: matrix.script == 'test-coverage' && always()
        run: .github/files/coverage-munger/process-coverage.sh

      - name: Check for artifacts
        id: check-artifacts
        # Default for `if` is `success()`, we want this to run always.
        if: always()
        run: |
          [[ -d artifacts ]] && find artifacts -type d -empty -delete
          if [[ -d artifacts ]]; then
            echo "any=true" >> "$GITHUB_OUTPUT"
          else
            echo "any=false" >> "$GITHUB_OUTPUT"
          fi
      - name: Upload artifacts
        id: upload-artifacts
        if: always() && steps.check-artifacts.outputs.any == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.artifact }}
          path: artifacts
          include-hidden-files: true
          retention-days: 7

  publish-coverage-data:
    name: Publish coverage data
    runs-on: ubuntu-latest
    timeout-minutes: 10 # 2025-01-10 Wild guess
    needs: run-tests
    if: always() && needs.run-tests.outputs.did-coverage == 'true' && ( github.event_name == 'pull_request' || github.ref == 'refs/heads/trunk' )
    steps:
      - uses: actions/checkout@v4

      - name: Setup tools
        uses: ./.github/actions/tool-setup

      - name: Download coverage artifact
        uses: actions/download-artifact@v4
        with:
          name: 'Code coverage'
          path: coverage

      - name: Upload coverage results
        env:
          PR_ID: ${{ github.event_name != 'pull_request' && 'trunk' || github.event.pull_request.number }}
          SECRET: ${{ secrets.CODECOV_SECRET }}
          STATUS: ${{ needs.run-tests.outputs.coverage-status }}
          PR_HEAD: ${{ github.event.pull_request.head.sha }}
          API_TOKEN_GITHUB: ${{ secrets.GITHUB_TOKEN }}
        run: .github/files/coverage-munger/upload-coverage.sh

  storybook-test:
    name: Storybook tests
    runs-on: ubuntu-latest
    timeout-minutes: 20 # 2024-02-23 Wild guess
    steps:
      - uses: actions/checkout@v4

      # For pull requests, list-changed-projects.sh needs the merge base.
      # But it doesn't have to be checked out.
      - name: Deepen to merge base
        if: github.event_name == 'pull_request'
        uses: ./.github/actions/deepen-to-merge-base
        with:
          checkout: false

      - name: Setup tools
        uses: ./.github/actions/tool-setup

      - name: Monorepo install
        run: |
          echo "::group::Pnpm"
          pnpm install
          echo "::endgroup::"

      - name: Detect changed projects
        id: changed
        run: |
          CHANGED=$( .github/files/list-changed-projects.sh )
          PROJECTS=$( node -e 'const r = { "js-packages/storybook": true }; for ( const p of require( "./projects/js-packages/storybook/storybook/projects.js" ) ) { const m = p.match( /\/projects\/([^/]+\/[^/]+)(?:$|\/)/ ); m && ( r[ m[1] ] = true ); } console.log( JSON.stringify( r ) );' )

          ANY=$( jq --argjson changed "$CHANGED" --argjson projects "$PROJECTS" -n '$changed | with_entries( select( $projects[ .key ] ) ) | any' )
          echo "any=${ANY}" >> "$GITHUB_OUTPUT"

      - name: Build storybook
        if: steps.changed.outputs.any == 'true'
        run: |
          pnpm jetpack build -v js-packages/storybook

      - name: Install playwright
        if: steps.changed.outputs.any == 'true'
        run: |
          cd projects/js-packages/storybook
          pnpm exec playwright install --with-deps chromium

      - name: Test storybook
        if: steps.changed.outputs.any == 'true'
        run: |
          cd projects/js-packages/storybook
          node bin/webserver.mjs
          REFERENCE_URL=https://automattic.github.io/jetpack-storybook/ pnpm exec test-storybook -c storybook --url 'http://127.0.0.1:6006/index.html'

  # Probably this should be a linting test, but we don't run linting on trunk or release branches.
  plugin-deps:
    name: Check plugin monorepo dep versions
    runs-on: ubuntu-latest
    timeout-minutes: 2  # 2022-09-08: Should only take a few seconds.
    steps:
      - uses: actions/checkout@v4
      - name: Setup tools
        uses: ./.github/actions/tool-setup
        with:
          node: false
      - name: Run check
        run: |
          if [[ "$GITHUB_EVENT_NAME" == 'push' ]]; then
            REF="${GITHUB_REF#refs/heads/}"
          elif [[ "$GITHUB_EVENT_NAME" == 'pull_request' || "$GITHUB_EVENT_NAME" == 'pull_request_target' ]]; then
            REF="$GITHUB_BASE_REF"
          else
            echo "::error::Unsupported github event \"$GITHUB_EVENT_NAME\""
            exit 1
          fi
          echo "Detected target ref \"$REF\""

          if [[ "$REF" == trunk ]]; then
            ARGS=( --dev )
          elif [[ "$REF" == */branch-* ]]; then
            ARGS=( --release )
            TMP="$(jq -r --arg P "${REF%%/branch-*}" '.extra["release-branch-prefix"] | if type == "array" then . else [ . ] end | if index( $P ) then input_filename | match( "^projects/plugins/([^/]+)/composer.json$" ).captures[0].string else empty end' projects/plugins/*/composer.json)"
            while IFS= read -r LINE; do
              ARGS+=( "$LINE" )
            done <<<"$TMP"
          else
            echo "Unsupported ref \"$REF\", ignoring"
            exit 0
          fi

          echo "Running tools/check-plugin-monorepo-deps.sh ${ARGS[@]}"
          tools/check-plugin-monorepo-deps.sh "${ARGS[@]}"

  # Probably this should be a linting test too, but we don't run linting on trunk or release branches.
  phan:
    name: Static analysis
    runs-on: ubuntu-latest
    timeout-minutes: 20  # 2024-05-02: Up to about 8 minutes now that we're running against the old WP stubs too.
    steps:
      - uses: actions/checkout@v4
      - name: Setup tools
        uses: ./.github/actions/tool-setup
      - name: Pnpm install
        run: pnpm install
      - name: Run phan
        run: pnpm jetpack phan --all -v --update-baseline --format github
      - name: Run phan for previous WP version too
        env:
          # Don't bother complaining about unused suppressions that may be used with the newer stubs. See .phan/config.base.php for how this gets applied.
          NO_PHAN_UNUSED_SUPPRESSION: 1
        run: |
          composer update --prefer-lowest php-stubs/wordpress-stubs php-stubs/wordpress-tests-stubs
          # Don't re-update baselines here, only check.
          pnpm jetpack phan --all -v --format github
      - name: Check baselines
        run: |
          # Anything changed? (with a side of printing the diff)
          if git diff --exit-code --ignore-matching-lines='^    // ' -- .phan/baseline.php '*/.phan/baseline.php'; then
            exit 0
          fi

          # Collect which projects changed to suggest the right command.
          PROJECTS=()
          for f in $( git -c core.quotepath=off diff --name-only -- .phan/baseline.php '*/.phan/baseline.php' ); do
            # --name-only and --ignore-matching-lines don't combine, so we have to do the check separately.
            if git diff --quiet --exit-code --ignore-matching-lines='^    // ' -- "$f"; then
              continue
            fi

            if [[ "$f" == ".phan/baseline.php" ]]; then
              SLUG=monorepo
            elif [[ "$f" == projects/*/*/.phan/baseline.php ]]; then
              SLUG=${f%/.phan/baseline.php}
              SLUG=${SLUG#projects/}
            elif SLUG=$( grep -v '^[ \t]*\/\/' .phan/monorepo-pseudo-projects.jsonc | jq -re --arg f "${f%.phan/baseline.php}" 'to_entries[] | select( .value == $f ) | .key' ); then
              : # Ok
            else
              SLUG=
            fi
            if grep -q 'This baseline has no suppressions' "$f"; then
              if [[ -n "$SLUG" ]]; then
                echo "::error file=$f::This Phan baseline is now empty (good job!). You may remove it, or if you want to keep it (e.g. if you expect new unfixed issues to be added in the future) you can run \`jetpack phan --update-baseline $SLUG\` to update it."
              else
                echo "::error file=$f::This Phan baseline is now empty (good job!). You may remove it."
              fi
            elif [[ -n "$SLUG" ]]; then
              PROJECTS+=( "$SLUG" )
            else
              echo "::error file=$f::This Phan baseline has changed and should be updated. This Action was unable to determine the command needed to update it; please report this to the Garage team."
            fi
          done
          if [[ ${#PROJECTS[@]} -gt 0 ]]; then
            echo "::error::Phan baselines have changed (good job!). Run \`jetpack phan --update-baseline ${PROJECTS[*]}\` to update them."
          fi
          exit 1