main.yml

name: Main

on:
  push:
    branches:
      - develop
    tags:
      - v*
  pull_request:
    branches:
      - develop
  repository_dispatch:
    types: [rebuild]
  workflow_dispatch:
    inputs:
      skip-test:
        description: "Skip test"
        type: choice
        required: true
        default: "false"
        options:
          - "true"
          - "false"
      plugin-version:
        description: "Plugin version"
        required: false
        type: string
        default: "LATEST"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
env:
  JAVA_VERSION: "21"
  DOCKER_APT_PACKAGES: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip
  DOCKER_PYTHON_LIBRARIES: kestra
  PLUGIN_VERSION: ${{ github.event.inputs.plugin-version != null && github.event.inputs.plugin-version || 'LATEST' }}
  # to save corepack from itself
  COREPACK_INTEGRITY_KEYS: 0
jobs:
  build-artifacts:
    name: Build Artifacts
    runs-on: ubuntu-latest
    timeout-minutes: 60
    outputs:
      docker-tag: ${{ steps.vars.outputs.tag }}
      docker-artifact-name: ${{ steps.vars.outputs.artifact }}
      plugins: ${{ steps.plugins-list.outputs.plugins }}
    steps:

      - name: Checkout - Current ref
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Checkout - GitHub Actions
        uses: actions/checkout@v4
        with:
          repository: kestra-io/actions
          path: actions
          ref: main

      # Npm
      - name: Setup - Npm install
        shell: bash
        working-directory: ui
        run: npm ci

      # Setup build
      - name: Setup - Build
        uses: ./actions/.github/actions/setup-build
        id: build
        with:
          java-enabled: true
          node-enabled: true
          caches-enabled: true

      # Get Plugins List
      - name: Plugins - Get List
        uses: ./.github/actions/plugins-list
        if: "!startsWith(github.ref, 'refs/tags/v')"
        id: plugins-list
        with:
          plugin-version: ${{ env.PLUGIN_VERSION }}

      # Set Plugins List
      - name: Plugins - Set List
        id: plugins
        if: "!startsWith(github.ref, 'refs/tags/v')"
        run: |
          PLUGINS="${{ steps.plugins-list.outputs.plugins }}"
          TAG=${GITHUB_REF#refs/*/}
          if [[ $TAG = "master" || $TAG == v* ]]; then
            echo "plugins=$PLUGINS" >> $GITHUB_OUTPUT
          else
            echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots $PLUGINS" >> $GITHUB_OUTPUT
          fi

      # Docker Tag
      - name: Setup - Docker vars
        id: vars
        run: |
          TAG=${GITHUB_REF#refs/*/}
          if [[ $TAG = "master" ]]
          then
            TAG="latest";
          elif [[ $TAG = "develop" ]]
          then
            TAG="develop";
          elif [[ $TAG = v* ]]
          then
            TAG="${TAG}";
          else
            TAG="build-${{ github.run_id }}";
          fi
          echo "tag=${TAG}" >> $GITHUB_OUTPUT
          echo "artifact=docker-kestra-${TAG}" >> $GITHUB_OUTPUT

      # Build
      - name: Gradle - Build
        run: |
          ./gradlew executableJar

      - name: Artifacts - Copy exe to image
        run: |
          cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra

      # Docker setup
      - name: Docker - Setup QEMU
        uses: docker/setup-qemu-action@v3

      - name: Docker - Setup Buildx
        uses: docker/setup-buildx-action@v3

      # Docker Build
      - name: Docker - Build & export image
        uses: docker/build-push-action@v6
        if: "!startsWith(github.ref, 'refs/tags/v')"
        with:
          context: .
          push: false
          file: Dockerfile
          tags: |
            kestra/kestra:${{ steps.vars.outputs.tag }}
          build-args: |
            KESTRA_PLUGINS=${{ steps.plugins.outputs.plugins }}
            APT_PACKAGES=${{ env.DOCKER_APT_PACKAGES }}
            PYTHON_LIBRARIES=${{ env.DOCKER_PYTHON_LIBRARIES }}
          outputs: type=docker,dest=/tmp/${{ steps.vars.outputs.artifact }}.tar

      # Upload artifacts
      - name: Artifacts - Upload JAR
        uses: actions/upload-artifact@v4
        with:
          name: jar
          path: build/libs/

      - name: Artifacts - Upload Executable
        uses: actions/upload-artifact@v4
        with:
          name: exe
          path: build/executable/

      - name: Artifacts - Upload Docker
        uses: actions/upload-artifact@v4
        if: "!startsWith(github.ref, 'refs/tags/v')"
        with:
          name: ${{ steps.vars.outputs.artifact }}
          path: /tmp/${{ steps.vars.outputs.artifact }}.tar

  check-frontend:
    name: Run frontend tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout - Current ref
        uses: actions/checkout@v4

      - name: Npm - Install
        shell: bash
        working-directory: ui
        run: npm ci

      - name: Npm - Lint
        uses: reviewdog/action-eslint@v1
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          reporter: github-pr-review # Change reporter.
          workdir: "ui"

      - name: Npm - Run unit tests
        shell: bash
        working-directory: ui
        run: npm run test:unit

      - name: Npm - Run build
        shell: bash
        working-directory: ui
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        run: npm run build

      - name: Storybook - Install Playwright
        shell: bash
        working-directory: ui
        run: npx playwright install --with-deps

      - name: Storybook - Build
        shell: bash
        working-directory: ui
        run: npm run build-storybook --quiet

      - name: Storybook - Run tests
        shell: bash
        working-directory: ui
        run: |
          npx concurrently -k -s first -n "SB,TEST" -c "magenta,blue" \
            "npx http-server storybook-static --port 6006 --silent" \
            "npx wait-on tcp:127.0.0.1:6006 && npm run test-storybook"

  # Run Reusable Workflow from QA repository
  check-e2e:
    name: Check E2E Tests
    needs: build-artifacts
    if: ${{ (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') && !startsWith(github.ref, 'refs/tags/v') }}
    uses: ./.github/workflows/e2e.yml
    strategy:
      fail-fast: false
      matrix:
        backends: ["postgres"]
    with:
      tags: oss
      docker-artifact-name: ${{ needs.build-artifacts.outputs.docker-artifact-name }}
      docker-image-tag: kestra/kestra:${{ needs.build-artifacts.outputs.docker-tag }}
      backend: ${{ matrix.backends }}
    secrets:
      GITHUB_AUTH_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }}
      GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}

  check:
    env:
      SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
      GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
    name: Check & Publish
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      # Checkout
      - name: Checkout - Current ref
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # Checkout GitHub Actions
      - name: Checkout - GitHub Actions
        uses: actions/checkout@v4
        with:
          repository: kestra-io/actions
          path: actions
          ref: main

      # Setup build
      - name: Setup - Build
        uses: ./actions/.github/actions/setup-build
        id: build
        with:
          java-enabled: true
          node-enabled: true
          python-enabled: true
          caches-enabled: true

      # Services
      - name: Setup - Start docker compose
        run: docker compose -f docker-compose-ci.yml up -d
        if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}

      # Gradle check
      - name: Gradle - Build
        if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }}
        env:
          GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
        run: |
          echo $GOOGLE_SERVICE_ACCOUNT | base64 -d > ~/.gcp-service-account.json
          export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.gcp-service-account.json
          ./gradlew check javadoc --parallel

      # report test
      - name: Test - Publish Test Results
        uses: dorny/test-reporter@v1
        if: always()
        with:
          name: Java Tests Report
          reporter: java-junit
          path: '**/build/test-results/test/TEST-*.xml'
          list-suites: 'failed'
          list-tests: 'failed'
          fail-on-error: 'false'

      # Sonar
      - name: Test - Analyze with Sonar
        if: ${{ env.SONAR_TOKEN != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
        run: ./gradlew sonar --info

      # GCP
      - name: GCP - Auth with unit test account
        id: auth
        if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 }}
        continue-on-error: true
        uses: "google-github-actions/auth@v2"
        with:
          credentials_json: "${{ secrets.GOOGLE_SERVICE_ACCOUNT }}"

      - name: GCP - Setup Cloud SDK
        if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
        uses: "google-github-actions/setup-gcloud@v2"

      # Allure check
      - name: Allure - Generate slug variables
        uses: rlespinasse/github-slug-action@v5

      - name: Allure - Publish report
        uses: andrcuns/allure-publish-action@v2.9.0
        if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
        continue-on-error: true
        env:
          GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          JAVA_HOME: /usr/lib/jvm/default-jvm/
        with:
          storageType: gcs
          resultsGlob: "**/build/allure-results"
          bucket: internal-kestra-host
          baseUrl: "https://internal.dev.kestra.io"
          prefix: ${{ format('{0}/{1}', github.repository, 'allure/java') }}
          copyLatest: true
          ignoreMissingResults: true

      # Jacoco
      - name: Jacoco - Copy reports
        if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
        continue-on-error: true
        run: |
          mv build/reports/jacoco/testCodeCoverageReport build/reports/jacoco/test/
          mv build/reports/jacoco/test/testCodeCoverageReport.xml build/reports/jacoco/test/jacocoTestReport.xml
          gsutil -m rsync -d -r  build/reports/jacoco/test/ gs://internal-kestra-host/${{ format('{0}/{1}', github.repository, 'jacoco') }}

      # Codecov
      - name: Codecov - Upload coverage reports
        uses: codecov/codecov-action@v5
        if: ${{ !cancelled() && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
        continue-on-error: true
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: Codecov - Upload test results
        uses: codecov/test-results-action@v1
        if: ${{ !cancelled() && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }}
        continue-on-error: true
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

  release:
    name: Github Release
    runs-on: ubuntu-latest
    needs: [check, check-e2e]
    if: |
      always() &&
      startsWith(github.ref, 'refs/tags/v') &&
      needs.check.result == 'success' &&
      (
        github.event.inputs.skip-test == 'true' ||
        (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success')
      )
    steps:
      # Download Exec
      - name: Artifacts - Download executable
        uses: actions/download-artifact@v4
        if: startsWith(github.ref, 'refs/tags/v')
        with:
          name: exe
          path: build/executable

      # GitHub Release
      - name: GitHub - Create release
        id: create_github_release
        uses: "marvinpinto/action-automatic-releases@latest"
        if: startsWith(github.ref, 'refs/tags/v')
        continue-on-error: true
        with:
          repo_token: "${{ secrets.GITHUB_TOKEN }}"
          prerelease: false
          files: |
            build/executable/*

      # Trigger gha workflow to bump helm chart version
      - name: GitHub - Trigger the Helm chart version bump
        uses: peter-evans/repository-dispatch@v3
        if: steps.create_github_release.conclusion == 'success'
        with:
          token: ${{ secrets.GH_PERSONAL_TOKEN }}
          repository: kestra-io/helm-charts
          event-type: update-helm-chart-version
          client-payload: |-
            {
              "new_version": "${{ github.ref_name }}",
              "github_repository": "${{ github.repository }}",
              "github_actor": "${{ github.actor }}"
            }

  docker:
    name: Publish Docker
    runs-on: ubuntu-latest
    needs: [build-artifacts, check, check-e2e]
    if: |
      always() &&
      github.ref == 'refs/heads/develop' &&
      needs.check.result == 'success' &&
      (
        github.event.inputs.skip-test == 'true' ||
        (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success')
      )
    strategy:
      matrix:
        image:
          - tag: ${{needs.build-artifacts.outputs.docker-tag}}-no-plugins
            packages: ""
            python-libraries: ""

          - tag: ${{needs.build-artifacts.outputs.docker-tag}}
            plugins: ${{needs.build-artifacts.outputs.plugins}}
            packages: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip
            python-libraries: kestra
    steps:
      - name: Checkout - Current ref
        uses: actions/checkout@v4

      # Docker setup
      - name: Docker - Setup QEMU
        uses: docker/setup-qemu-action@v3

      - name: Docker - Setup Docker Buildx
        uses: docker/setup-buildx-action@v3

      # Docker Login
      - name: Docker - Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_PASSWORD }}

      # Vars
      - name: Docker - Set image name
        id: vars
        run: |
          TAG=${GITHUB_REF#refs/*/}
          if [[ $TAG = "master" || $TAG == v* ]]; then
            echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
          else
            echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots ${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT
          fi

      # Build Docker Image
      - name: Artifacts - Download executable
        uses: actions/download-artifact@v4
        with:
          name: exe
          path: build/executable

      - name: Docker - Copy exe to image
        run: |
          cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra

      # Docker Build and push
      - name: Docker - Build image
        uses: docker/build-push-action@v6
        with:
          context: .
          push: true
          tags: kestra/kestra:${{ matrix.image.tag }}
          platforms: linux/amd64,linux/arm64
          build-args: |
            KESTRA_PLUGINS=${{ steps.vars.outputs.plugins }}
            APT_PACKAGES=${{matrix.image.packages}}
            PYTHON_LIBRARIES=${{matrix.image.python-libraries}}

  maven:
    name: Publish to Maven
    runs-on: ubuntu-latest
    needs: [check, check-e2e]
    if: |
      always() &&
      (github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v')) &&
      needs.check.result == 'success' &&
      (
        github.event.inputs.skip-test == 'true' ||
        (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success')
      )
    steps:
      - name: Checkout - Current ref
        uses: actions/checkout@v4

      # Checkout GitHub Actions
      - name: Checkout - GitHub Actions
        uses: actions/checkout@v4
        with:
          repository: kestra-io/actions
          path: actions
          ref: main

      # Setup build
      - name: Setup - Build
        uses: ./actions/.github/actions/setup-build
        id: build
        with:
          java-enabled: true
          node-enabled: true
          caches-enabled: true

      # Publish
      - name: Publish - Release package to Maven Central
        if: github.ref == 'refs/heads/develop'
        env:
          ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
          ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
          SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
          SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
          SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
        run: |
          mkdir -p ~/.gradle/
          echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
          echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
          echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
          echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
          ./gradlew publishToSonatype ${{ startsWith(github.ref, 'refs/tags/v')  && 'closeAndReleaseSonatypeStagingRepository' || '' }}

      # Gradle dependency
      - name: Java - Gradle dependency graph
        if: ${{ github.ref == 'refs/heads/develop' }}
        uses: gradle/actions/dependency-submission@v4

  end:
    runs-on: ubuntu-latest
    needs:
      - check-e2e
      - check
      - maven
      - docker
      - release
    if: always()
    env:
      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
    steps:
      # Update
      - name: Github - Update internal
        uses: benc-uk/workflow-dispatch@v1
        if: github.ref == 'refs/heads/develop' && needs.docker.result == 'success'
        with:
          workflow: oss-build.yml
          repo: kestra-io/infra
          ref: master
          token: ${{ secrets.GH_PERSONAL_TOKEN }}

      # Slack
      - name: Slack - Notification
        uses: Gamesight/slack-workflow-status@master
        if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }}
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
          name: GitHub Actions
          icon_emoji: ":github-actions:"
          channel: "C02DQ1A7JLR" # _int_git channel