pre-merge.yaml

name: Pre merge

concurrency:
  # This stops any build on a branch if a new commit is pushed on that branch
  # the second check is here to run each build when pushing on the branch `q/*`
  # in order for bert-e to merge any branch that passes the queue before waiting
  # for the entire queue to passe the build.
  # if the branch is 'q/*' then we append the commit sha to the concurrency group to make it unique.
  # this works because the '||' or condition will only evaluate the right side if the left side is false ;-)
  group: "tests-${{ github.ref_name }}-${{ ( !startsWith(github.ref_name, 'q/') || github.sha) }}"
  cancel-in-progress: true

on:
  push:
    branches:
      - "release/**"
      - "feature/**"
      - "improvement/**"
      - "hotfix/**"
      - "bugfix/**"
      - "documentation/**"
      - "user/**"
      - "dependabot/**"
      - "w/**"
      - "q/*/**"

jobs:
  changed-files:
    runs-on: ubuntu-24.04
    outputs:
      docs: ${{ steps.diff.outputs.docs_any_modified == 'true' }}
      shell-ui: ${{ steps.diff.outputs.shell-ui_any_modified == 'true' }}
      unit-tests-ui: ${{ steps.diff.outputs.unit-tests-ui_any_modified == 'true' }}
      unit-tests-shell-ui: ${{ steps.diff.outputs.unit-tests-shell-ui_any_modified == 'true' }}
      unit-tests-crd-client-generator: ${{ steps.diff.outputs.unit-tests-crd-client-generator_any_modified == 'true' }}
      unit-tests-metalk8s-operator: ${{ steps.diff.outputs.unit-tests-metalk8s-operator_any_modified == 'true' }}
      unit-tests-storage-operator: ${{ steps.diff.outputs.unit-tests-storage-operator_any_modified == 'true' }}
      unit-tests-salt: ${{ steps.diff.outputs.unit-tests-salt_any_modified == 'true' }}
      unit-tests-lib-alert-tree: ${{ steps.diff.outputs.unit-tests-lib-alert-tree_any_modified == 'true' }}
      integration-tests-ui: ${{ steps.diff.outputs.integration-tests-ui_any_modified == 'true' }}
      # If the workflow is triggered on a development branch let's always build
      build: ${{ startsWith(github.ref_name, 'development/') || steps.diff.outputs.build_any_modified == 'true' }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Get base sha
        run: |
          source VERSION
          branch=development/${VERSION_MAJOR}.${VERSION_MINOR}
          if [[ ! $(git ls-remote --heads origin $branch) ]]; then
            branch=development/${VERSION_MAJOR}
          fi
          echo "BASE_SHA=$(git merge-base HEAD origin/$branch)" >> $GITHUB_ENV
      - name: Get changed files
        id: diff
        uses: tj-actions/changed-files@v45
        with:
          base_sha: ${{ env.BASE_SHA }}
          files_yaml_from_source_file: .github/changed-files.yaml
      - name: List all changed files
        env:
          ALL_DOCS_CHANGED_FILES: ${{ steps.diff.outputs.docs_all_changed_files }}
          ALL_SHELL_UI_CHANGED_FILES: ${{ steps.diff.outputs.shell-ui_all_changed_files }}
          ALL_UNIT_TESTS_UI_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-ui_all_changed_files }}
          ALL_UNIT_TESTS_SHELL_UI_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-shell-ui_all_changed_files }}
          ALL_UNIT_TESTS_CRD_CLIENT_GENERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-crd-client-generator_all_changed_files }}
          ALL_UNIT_TESTS_METALK8S_OPERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-metalk8s-operator_all_changed_files }}
          ALL_UNIT_TESTS_STORAGE_OPERATOR_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-storage-operator_all_changed_files }}
          ALL_UNIT_TESTS_SALT_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-salt_all_changed_files }}
          ALL_UNIT_TESTS_LIB_ALERT_TREE_CHANGED_FILES: ${{ steps.diff.outputs.unit-tests-lib-alert-tree_all_changed_files }}
          ALL_INTEGRATION_TESTS_UI_CHANGED_FILES: ${{ steps.diff.outputs.integration-tests-ui_all_changed_files }}
          ALL_BUILD_CHANGED_FILES: ${{ steps.diff.outputs.build_all_changed_files }}
        run: |
          for type in docs shell-ui unit-tests-ui unit-tests-shell-ui unit-tests-crd-client-generator unit-tests-metalk8s-operator unit-tests-storage-operator unit-tests-salt unit-tests-lib-alert-tree integration-tests-ui build; do
            echo "All $type changed files:"
            changed_files_var=$(echo "ALL_${type^^}_CHANGED_FILES" | tr '-' '_')
            echo "${!changed_files_var}"
          done

  build:
    needs: changed-files
    if: needs.changed-files.outputs.build == 'true'
    uses: ./.github/workflows/build.yaml
    secrets: inherit

  build-docs:
    needs: changed-files
    # Build of docs is triggered by "build" so do not re-trigger it
    if: needs.changed-files.outputs.docs == 'true' && needs.changed-files.outputs.build != 'true'
    uses: ./.github/workflows/build-docs.yaml
    secrets: inherit

  build-shell-ui:
    needs: changed-files
    # Build of shell-ui is triggered by "build" so do not re-trigger it
    if: needs.changed-files.outputs.shell-ui == 'true' && needs.changed-files.outputs.build != 'true'
    uses: ./.github/workflows/build-shell-ui.yaml
    secrets: inherit

  e2e-tests:
    needs:
      - build
      - changed-files
    if: needs.changed-files.outputs.build == 'true'
    uses: ./.github/workflows/e2e-tests.yaml
    secrets: inherit
    with:
      artifacts-url: ${{ needs.build.outputs.artifact-link }}

  lint:
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Python 3
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      # NOTE: Today python3.6 is needed by some lint steps
      - name: Install Python 3.6
        uses: actions/setup-python@v5
        with:
          python-version: "3.6"
      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.20.14"
      - name: Install deps
        run: |
          export DEBIAN_FRONTEND=noninteractive
          sudo apt-get update
          sudo apt-get install --no-install-recommends -y shellcheck python2.7-minimal
          python3.10 -m pip install tox~=4.0.11
      - name: Install Helm
        env:
          HELM_VERSION: "3.10.0"
        run: |
          sudo curl -O https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz \
              && sudo tar -zxvf helm-v${HELM_VERSION}-linux-amd64.tar.gz \
              && sudo mv linux-amd64/helm /usr/local/bin/helm
      - name: Run all linting targets
        run: ./doit.sh lint

  unit_tests_ui:
    needs: changed-files
    if: needs.changed-files.outputs.unit-tests-ui == 'true'
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install node
        uses: actions/setup-node@v4
        with:
          node-version: "20"
      - name: Cache node modules
        id: cache-npm
        uses: actions/cache@v4
        env:
          cache-name: cache-node-modules
        with:
          # npm cache files are stored in `~/.npm` on Linux/macOS
          path: ~/.npm
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
            ${{ runner.os }}-
      - name: Install UI dependencies
        run: |
          cd ui
          npm ci --prefer-offline --no-audit --legacy-peer-deps --unsafe-perm --no-optional
      - name: Run all UI unit tests
        run: |
          cd ui
          npm run test:nowatch --no-update-notifier
      - name: Prepare artifacts
        run: mkdir -p "artifacts/ui" && mv ui/junit artifacts/ui/
      - name: Upload artifacts
        uses: scality/action-artifacts@v4
        with:
          method: upload
          url: https://artifacts.scality.net
          user: ${{ secrets.ARTIFACTS_USER }}
          password: ${{ secrets.ARTIFACTS_PASSWORD }}
          source: artifacts

  unit_tests_shell_ui:
    needs: changed-files
    if: needs.changed-files.outputs.unit-tests-shell-ui == 'true'
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install node
        uses: actions/setup-node@v4
        with:
          node-version: "20"
      - name: Cache node modules
        id: cache-npm
        uses: actions/cache@v4
        env:
          cache-name: cache-node-modules
        with:
          # npm cache files are stored in `~/.npm` on Linux/macOS
          path: ~/.npm
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
            ${{ runner.os }}-
      - name: Install UI dependencies
        run: |
          cd shell-ui
          npm ci --prefer-offline --no-audit --legacy-peer-deps -d
      - name: Run all Shell UI unit tests
        run: |
          cd shell-ui
          npm run test --no-update-notifier

  unit_tests_crd_client_generator:
    needs: changed-files
    if: needs.changed-files.outputs.unit-tests-crd-client-generator == 'true'
    runs-on: ubuntu-20.04
    defaults:
      run:
        working-directory: "tools/crd-client-generator-js"
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install node
        uses: actions/setup-node@v4
        with:
          node-version: "20"
      - name: Cache node modules
        uses: actions/cache@v4
        env:
          cache-name: cache-node-modules
        with:
          # npm cache files are stored in `~/.npm` on Linux/macOS
          path: ~/.npm
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
            ${{ runner.os }}-
      - name: Install CRD client generator dependencies
        run: npm ci --prefer-offline --no-audit
      - name: Run all CRD client generator unit tests
        run: npm run test --no-update-notifier

  unit_tests_metalk8s_operator:
    needs: changed-files
    if: needs.changed-files.outputs.unit-tests-metalk8s-operator == 'true'
    runs-on: ubuntu-20.04
    defaults:
      run:
        working-directory: "operator"
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.20.14"
      - name: Cache Go modules
        uses: actions/cache@v4
        env:
          cache-name: cache-go-modules
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('go.sum') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
            ${{ runner.os }}-
      - name: Run all MetalK8s operator unit and integration tests
        run: make test

  unit_tests_storage_operator:
    needs: changed-files
    if: needs.changed-files.outputs.unit-tests-storage-operator == 'true'
    runs-on: ubuntu-20.04
    defaults:
      run:
        working-directory: "storage-operator"
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.20.14"
      - name: Cache Go modules
        uses: actions/cache@v4
        env:
          cache-name: cache-go-modules
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('go.sum') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
            ${{ runner.os }}-
      - name: Run all storage-operator unit and integration tests
        run: make test

  unit_tests_salt:
    needs: changed-files
    if: needs.changed-files.outputs.unit-tests-salt == 'true'
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Python 3.6
        uses: actions/setup-python@v5
        with:
          python-version: "3.6"
      - name: Install deps
        run: |
          pip install tox
      - name: Run all salt unit tests
        run: tox -e unit-tests

  unit_tests_lib_alert_tree:
    needs: changed-files
    if: needs.changed-files.outputs.unit-tests-lib-alert-tree == 'true'
    runs-on: ubuntu-20.04
    defaults:
      run:
        working-directory: "tools/lib-alert-tree"
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Python 3
        uses: actions/setup-python@v5
        with:
          python-version: "3.6"
      - name: Install deps
        run: python3.6 -m pip install poetry~=1.1.9
      - name: Install lib_alert_tree
        run: poetry install -E cli
      - name: Run all lib_alert_tree unit tests
        run: poetry run pytest -vv

  build_integration_container_nginx:
    needs: changed-files
    if: needs.changed-files.outputs.integration-tests-ui == 'true'
    runs-on: ubuntu-20.04
    permissions: write-all
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Install node
        uses: actions/setup-node@v4
        with:
          node-version: "20"
      - name: Cache node modules
        id: cache-npm
        uses: actions/cache@v4
        env:
          cache-name: cache-node-modules
        with:
          # npm cache files are stored in `~/.npm` on Linux/macOS
          path: ~/.npm
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
            ${{ runner.os }}-
      - name: Install Shell UI dependencies
        run: |
          cd shell-ui
          npm ci --legacy-peer-deps
      - name: Build Shell UI
        run: |
          cd shell-ui
          npm run rsbuild
      - name: Install UI dependencies
        run: |
          cd ui
          npm ci --legacy-peer-deps
      - name: Build UI
        run: |
          cd ui
          npm run build
      - name: Prepare Services Volumes
        run: |
          mkdir -p service
          mkdir -p service/shell
          cp -r ui/build/* service/
          cp -r ui/build/.well-known service/.well-known
          cp -r shell-ui/build/* service/shell/
      - name: Build container with nginx configuration for integration tests
        run: |
          cat > Dockerfile <<EOF
          FROM nginx:alpine
          COPY ui/standalone-nginx.conf /etc/nginx/conf.d/default.conf
          ADD service /usr/share/nginx/html
          EOF
      - name: Login to Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - uses: docker/build-push-action@v6
        with:
          push: true
          file: Dockerfile
          context: .
          tags: ghcr.io/${{ github.repository_owner }}/metalk8s-nginx-integration-tests:${{ github.sha }}

  integration_tests_ui:
    runs-on: ubuntu-20.04
    needs:
      - build_integration_container_nginx
      - changed-files
    if: needs.changed-files.outputs.integration-tests-ui == 'true'
    services:
      app:
        image: ghcr.io/${{ github.repository_owner }}/metalk8s-nginx-integration-tests:${{ github.sha }}
        credentials:
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
        ports:
          - 80:80
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install node
        uses: actions/setup-node@v4
        with:
          node-version: "20"
      - name: Cache node modules
        id: cache-npm
        uses: actions/cache@v4
        env:
          cache-name: cache-node-modules
        with:
          # npm cache files are stored in `~/.npm` on Linux/macOS
          path: ~/.npm
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
            ${{ runner.os }}-
      - name: Install Cypress and its dependencies
        run: |
          cd ui
          PKGS="har-validator cypress cypress-cucumber-preprocessor cypress-wait-until @testing-library/cypress"
          for pkg in $PKGS; do
            npm install --no-save --no-package-lock --legacy-peer-deps $pkg@$(node -p \
              -e "require('./package-lock.json').dependencies['$pkg'].version" \
            ) || exit 1
          done
      - name: Wait for application to be available
        run: |
          bash -c '
            attempts=0
            until curl -Isfo /dev/null http://localhost/; do
              (( attempts++ ))
              if [ $attempts -gt 100 ]; then
                >&2 echo "Failed to reach application after 5 minutes"
                exit 1
              fi
              sleep 3
            done
          '
      - name: Run all UI integration tests
        env:
          CYPRESS_BASE_URL: http://localhost
        run: |
          cd ui
          rm -rf babel.config.js
          npm run test:integration --no-update-notifier
      - name: Prepare upload folder
        if: always()
        run: |
          mkdir -p upload/ui/cypress
          mv ui/cypress/screenshots upload/ui/cypress/screenshots || true
          mv ui/cypress/videos upload/ui/cypress/videos || true
          mv junit upload/ui || true
      - name: Upload artifacts
        if: always()
        uses: scality/action-artifacts@v4
        with:
          method: upload
          url: https://artifacts.scality.net
          user: ${{ secrets.ARTIFACTS_USER }}
          password: ${{ secrets.ARTIFACTS_PASSWORD }}
          source: upload

  write-final-status:
    runs-on: ubuntu-20.04
    needs:
      - changed-files
      - build-docs
      - build-shell-ui
      - e2e-tests
      - lint
      - unit_tests_ui
      - unit_tests_shell_ui
      - unit_tests_crd_client_generator
      - unit_tests_metalk8s_operator
      - unit_tests_storage_operator
      - unit_tests_salt
      - unit_tests_lib_alert_tree
      - integration_tests_ui
    if: always()
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Upload final status
        if: always()
        uses: scality/actions/upload_final_status@1.8.0
        with:
          ARTIFACTS_USER: ${{ secrets.ARTIFACTS_USER }}
          ARTIFACTS_PASSWORD: ${{ secrets.ARTIFACTS_PASSWORD }}
          JOBS_RESULTS: ${{ join(needs.*.result) }}