install diff-filter-buildkite-plugin to only build when relevant files changed

pipelines/main/launch_unsigned_jobs.yml

@@ -12,6 +12,25 @@
 # and only need to touch the webui configuration when we need to alter
 # something about the privileged steps.
 
+common:
+  - diff-filter-build_plugin: &diff-filter-build
+      https://github.com/fatteneder/diff-filter-buildkite-plugin#main:
+        name: "build"
+        ignore:
+          - "*.md"
+          - "*.json"
+          - "CITATION.*"
+          - ".gitignore"
+          - ".clangd"
+          - ".mailmap"
+        # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main
+        # when running tests in Julia-CI/julia-buildkite,
+        # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master
+        target_branch: "master"
+  - pr-labels_plugin: &pr-labels
+      sv-oss/github-pr-labels#v0.0.2:
+        publish-env-var: PULL_REQUEST_LABELS
+
 steps:
   - group: "Build"
     notify:
@@ -23,26 +42,34 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
+          - *pr-labels
         commands: |
+          FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
+          BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE}))
           # Launch Linux build jobs
           GROUP="Build" \
               ALLOW_FAIL="false" \
+              BUILD="$${BUILD}" \
               bash .buildkite/utilities/arches_pipeline_upload.sh \
               .buildkite/pipelines/main/platforms/build_linux.arches \
               .buildkite/pipelines/main/platforms/build_linux.yml
           # Launch macOS packaging jobs
           GROUP="Build" \
               ALLOW_FAIL="false" \
+              BUILD="$${BUILD}" \
               bash .buildkite/utilities/arches_pipeline_upload.sh \
               .buildkite/pipelines/main/platforms/build_macos.arches \
               .buildkite/pipelines/main/platforms/build_macos.yml
           GROUP="Build" \
               ALLOW_FAIL="false" \
+              BUILD="$${BUILD}" \
               bash .buildkite/utilities/arches_pipeline_upload.sh \
               .buildkite/pipelines/main/platforms/build_freebsd.arches \
               .buildkite/pipelines/main/platforms/build_freebsd.yml
           GROUP="Build" \
               ALLOW_FAIL="false" \
+              BUILD="$${BUILD}" \
               bash .buildkite/utilities/arches_pipeline_upload.sh \
               .buildkite/pipelines/main/platforms/build_windows.arches \
               .buildkite/pipelines/main/platforms/build_windows.yml
@@ -59,16 +86,22 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
+          - *pr-labels
         commands: |
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml
           buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
           buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
-          # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite
+          FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
+          BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE}))
+          if [[ $${BUILD} == 1 ]]; then
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
+            # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite
 
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml
+          fi
         agents:
           queue: "julia"
           os: "linux"
@@ -82,36 +115,42 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
+          - *pr-labels
         commands: |
           export ALLOW_FAIL="false"
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml
-          buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml
+          FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
+          BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE}))
+          if [[ $${BUILD} == 1 ]]; then
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml
+            buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml
 
-          # Launch Linux test jobs
-          GROUP="Test" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_linux.arches \
-              .buildkite/pipelines/main/platforms/test_linux.yml
-          # Launch i686-linux-gnu test jobs
-          GROUP="Test" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_linux.i686.arches \
-              .buildkite/pipelines/main/platforms/test_linux.i686.yml
-          # Launch macOS test jobs
-          GROUP="Test" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_macos.arches \
-              .buildkite/pipelines/main/platforms/test_macos.yml
-          # Launch FreeBSD test jobs
-          GROUP="Test" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_freebsd.arches \
-              .buildkite/pipelines/main/platforms/test_freebsd.yml
-          # Launch windows test jobs
-          GROUP="Test" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_windows.arches \
-              .buildkite/pipelines/main/platforms/test_windows.yml
+            # Launch Linux test jobs
+            GROUP="Test" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_linux.arches \
+                .buildkite/pipelines/main/platforms/test_linux.yml
+            # Launch i686-linux-gnu test jobs
+            GROUP="Test" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_linux.i686.arches \
+                .buildkite/pipelines/main/platforms/test_linux.i686.yml
+            # Launch macOS test jobs
+            GROUP="Test" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_macos.arches \
+                .buildkite/pipelines/main/platforms/test_macos.yml
+            # Launch FreeBSD test jobs
+            GROUP="Test" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_freebsd.arches \
+                .buildkite/pipelines/main/platforms/test_freebsd.yml
+            # Launch windows test jobs
+            GROUP="Test" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_windows.arches \
+                .buildkite/pipelines/main/platforms/test_windows.yml
+          fi
         agents:
           queue: "julia"
           os: "linux"
@@ -122,18 +161,24 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
+          - *pr-labels
         commands: |
           export ALLOW_FAIL="true"
-          # Launch Linux allowed-to-fail build jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/build_linux.yml
-          # Launch macOS allowed-to-fail build jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/build_macos.yml
+          FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
+          BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE}))
+          if [[ $${BUILD} == 1 ]]; then
+            # Launch Linux allowed-to-fail build jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/build_linux.yml
+            # Launch macOS allowed-to-fail build jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/build_macos.yml
+          fi
         agents:
           queue: "julia"
           os: "linux"
@@ -142,29 +187,34 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
+          - *pr-labels
         commands: |
           export ALLOW_FAIL="true"
-
-          # Launch Linux allowed-to-fail test jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/test_linux.yml
-          # Launch macOS allowed-to-fail test jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/test_macos.yml
-          # Launch FreeBSD allowed-to-fail jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/test_freebsd.yml
-          # Launch windows allowed-to-fail test jobs
-          GROUP="Allow Fail" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/test_windows.yml
+          FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
+          BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE}))
+          if [[ $${BUILD} == 1 ]]; then
+            # Launch Linux allowed-to-fail test jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/test_linux.yml
+            # Launch macOS allowed-to-fail test jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/test_macos.yml
+            # Launch FreeBSD allowed-to-fail jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/test_freebsd.yml
+            # Launch windows allowed-to-fail test jobs
+            GROUP="Allow Fail" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/test_windows.yml
+          fi
         agents:
           queue: "julia"
           os: "linux"

pipelines/main/launch_upload_jobs.yml

@@ -1,5 +1,24 @@
 # This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3
 
+common:
+  - diff-filter-build_plugin: &diff-filter-build
+      https://github.com/fatteneder/diff-filter-buildkite-plugin#main:
+        name: "build"
+        ignore:
+          - "*.md"
+          - "*.json"
+          - "CITATION.*"
+          - ".gitignore"
+          - ".clangd"
+          - ".mailmap"
+        # TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main
+        # when running tests in Julia-CI/julia-buildkite,
+        # but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master
+        target_branch: "master"
+  - pr-labels_plugin: &pr-labels
+      sv-oss/github-pr-labels#v0.0.2:
+        publish-env-var: PULL_REQUEST_LABELS
+
 steps:
   - group: "Upload"
     steps:
@@ -8,35 +27,40 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
+          - *pr-labels
         commands: |
           # Explicitly pass along the cryptic token to child pipelines
           export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
           export ALLOW_FAIL="false"
-
-          # Launch `upload_*` jobs to store tarballs into S3 once tests are done
-          GROUP="Upload" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/upload_linux.arches \
-              .buildkite/pipelines/main/platforms/upload_linux.yml
-          GROUP="Upload" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/upload_macos.arches \
-              .buildkite/pipelines/main/platforms/upload_macos.yml
-          GROUP="Upload" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/upload_freebsd.arches \
-              .buildkite/pipelines/main/platforms/upload_freebsd.yml
-          GROUP="Upload" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/upload_windows.arches \
-              .buildkite/pipelines/main/platforms/upload_windows.yml
-
-          for OS in linux macos freebsd windows; do \
-              GROUP="Upload" \
+          FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
+          BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE}))
+          if [[ $${BUILD} == 1 ]]; then
+            # Launch `upload_*` jobs to store tarballs into S3 once tests are done
+            GROUP="Upload" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/upload_linux.arches \
+                .buildkite/pipelines/main/platforms/upload_linux.yml
+            GROUP="Upload" \
                 bash .buildkite/utilities/arches_pipeline_upload.sh \
-                .buildkite/pipelines/main/platforms/upload_$${OS}.arches \
-                .buildkite/pipelines/main/misc/upload_buildkite_results.yml
-          done
+                .buildkite/pipelines/main/platforms/upload_macos.arches \
+                .buildkite/pipelines/main/platforms/upload_macos.yml
+            GROUP="Upload" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/upload_freebsd.arches \
+                .buildkite/pipelines/main/platforms/upload_freebsd.yml
+            GROUP="Upload" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/upload_windows.arches \
+                .buildkite/pipelines/main/platforms/upload_windows.yml
+
+            for OS in linux macos freebsd windows; do \
+                GROUP="Upload" \
+                  bash .buildkite/utilities/arches_pipeline_upload.sh \
+                  .buildkite/pipelines/main/platforms/upload_$${OS}.arches \
+                  .buildkite/pipelines/main/misc/upload_buildkite_results.yml
+            done
+          fi
 
           # Launch doctest deploy job
           buildkite-agent pipeline upload .buildkite/pipelines/main/misc/deploy_docs.yml
@@ -57,17 +81,23 @@ steps:
           - JuliaCI/external-buildkite#v1:
               version: "./.buildkite-external-version"
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
+          - *diff-filter-build
+          - *pr-labels
         commands: |
           # Explicitly pass along the cryptic token to child pipelines
           export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
+          FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
+          BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || $${FORCE}))
 
           export ALLOW_FAIL="true"
 
-          # Launch Linux allowed-to-fail upload jobs
-          GROUP="Allow Fail (Upload)" \
-              bash .buildkite/utilities/arches_pipeline_upload.sh \
-              .buildkite/pipelines/main/platforms/upload_linux.soft_fail.arches \
-              .buildkite/pipelines/main/platforms/upload_linux.yml
+          if [[ $${BUILD} == 1 ]]; then
+            # Launch Linux allowed-to-fail upload jobs
+            GROUP="Allow Fail (Upload)" \
+                bash .buildkite/utilities/arches_pipeline_upload.sh \
+                .buildkite/pipelines/main/platforms/upload_linux.soft_fail.arches \
+                .buildkite/pipelines/main/platforms/upload_linux.yml
+          fi
 
           # Don't share this with buildkite's env display
           unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET

pipelines/main/platforms/build_freebsd.yml

@@ -9,7 +9,10 @@ steps:
               repo_url: "https://github.com/JuliaCI/julia-buildkite"
         timeout_in_minutes: ${TIMEOUT?}
         soft_fail: ${ALLOW_FAIL?}
-        commands: "bash .buildkite/utilities/build_julia.sh"
+        commands: |
+          if [[ ${BUILD?} == 1 ]]; then
+            bash .buildkite/utilities/build_julia.sh
+          fi
         agents:
           queue: "julia"
           os: "freebsd"

pipelines/main/platforms/build_linux.yml

@@ -21,7 +21,11 @@ steps:
                 - "/cache/repos:/cache/repos"
         timeout_in_minutes: ${TIMEOUT?}
         soft_fail: ${ALLOW_FAIL?}
-        commands: "bash .buildkite/utilities/build_julia.sh"
+        commands: |
+          # always build on x86_64-linux-gnu so that we can at least build docs
+          if [[ ${BUILD?} == 1 || ${TRIPLET?} == "x86_64-linux-gnu" ]]; then
+            bash .buildkite/utilities/build_julia.sh
+          fi
         agents:
           queue: "julia"
           # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing