Add building and testing of PGO+LTO LLVM and Julia

pipelines/main/launch_unsigned_jobs.yml

@@ -30,6 +30,12 @@ steps:
               bash .buildkite/utilities/arches_pipeline_upload.sh \
               .buildkite/pipelines/main/platforms/build_linux.arches \
               .buildkite/pipelines/main/platforms/build_linux.yml
+          # Launch Linux PGO+LTO build jobs
+          GROUP="Build" \
+              ALLOW_FAIL="false" \
+              bash .buildkite/utilities/arches_pipeline_upload.sh \
+              .buildkite/pipelines/main/platforms/build_linux.pgo_lto.arches \
+              .buildkite/pipelines/main/platforms/build_linux.yml
           # Launch macOS packaging jobs
           GROUP="Build" \
               ALLOW_FAIL="false" \
@@ -97,6 +103,11 @@ steps:
               bash .buildkite/utilities/arches_pipeline_upload.sh \
               .buildkite/pipelines/main/platforms/test_linux.i686.arches \
               .buildkite/pipelines/main/platforms/test_linux.i686.yml
+          # Launch Linux PGO+LTO test jobs
+          GROUP="Test" \
+              bash .buildkite/utilities/arches_pipeline_upload.sh \
+              .buildkite/pipelines/main/platforms/test_linux.pgo_lto.arches \
+              .buildkite/pipelines/main/platforms/test_linux.yml
           # Launch macOS test jobs
           GROUP="Test" \
               bash .buildkite/utilities/arches_pipeline_upload.sh \

pipelines/main/platforms/build_linux.pgo_lto.arches

@@ -0,0 +1,8 @@
+# ROOTFS_IMAGE_NAME  TRIPLET                    ARCH     ARCH_ROOTFS    USE_JULIA_PGO_LTO    MAKE_FLAGS      TIMEOUT      ROOTFS_TAG   ROOTFS_HASH
+llvm_passes          x86_64-linux-gnu-pgo-lto   x86_64   x86_64         PGO_LTO              .               .            v7.6         ebdfa2d40c0e27842cccbfdd43710144f9adf9d8
+
+# These special lines allow us to embed default values for the columns above.
+# Any column without a default mapping here will simplpy substitute a `.` to the empty string
+
+# Builds should generally finish in much less time than this, but from-source builds can take longer
+#default TIMEOUT 80

pipelines/main/platforms/build_linux.yml

@@ -33,3 +33,4 @@ steps:
           TRIPLET: "${TRIPLET?}"
           MAKE_FLAGS: "${MAKE_FLAGS?}"
           ROOTFS_IMAGE_NAME: "${ROOTFS_IMAGE_NAME?}"
+          USE_JULIA_PGO_LTO: "${USE_JULIA_PGO_LTO}"

pipelines/main/platforms/test_linux.pgo_lto.arches

@@ -0,0 +1,10 @@
+# ROOTFS_IMAGE_NAME    TRIPLET                    ARCH           ARCH_ROOTFS    TIMEOUT    USE_RR     USE_JULIA_PGO_LTO     ROOTFS_TAG    ROOTFS_HASH
+tester_linux           x86_64-linux-gnu-pgo-lto   x86_64         x86_64         .          .          PGO_LTO               v6.00         770c0240be788cfeb9654e1980ea929d3ed98d1f
+
+# These special lines allow us to embed default values for the columns above.
+# Any column without a default mapping here will simply substitute a `.` to the empty string
+
+# Most tests should finish within ~45 minutes, barring exceptionally slow hardware
+# We double that to a default of 90 minutes, with an extra 45 minutes for cleanup,
+# including things like `rr` trace compression,
+#default TIMEOUT 135

utilities/build_julia.sh

@@ -28,6 +28,11 @@ echo "--- Collect make options"
 # These are the flags we'll provide to `make`
 MFLAGS=()
 
+if [[ ! -z "${USE_JULIA_PGO_LTO-}" ]]; then
+    MFLAGS+=( "-Ccontrib/pgo-lto" )
+    MFLAGS+=( "STAGE2_BUILD=$PWD" )
+fi
+
 # If we have the option, let's use `--output-sync`
 if ${MAKE} --help | grep output-sync >/dev/null 2>/dev/null; then
     MFLAGS+=( "--output-sync" )
@@ -96,3 +101,8 @@ fi
 
 echo "--- Upload build artifacts to buildkite"
 buildkite-agent artifact upload "${UPLOAD_FILENAME}.tar.gz"
+
+# Upload the profile data to allow for reproducible builds
+if [[ ! -z "${USE_JULIA_PGO_LTO-}" ]]; then
+    buildkite-agent artifact upload "contrib/pgo-lto/profiles/merged.prof"
+fi

utilities/extract_triplet.sh

@@ -26,6 +26,9 @@ case "${TRIPLET}" in
     *-gnunogpl) # builds that use `USE_GPL_LIBS=0`
         OS="linuxnogpl"
         ;;
+    *-gnu-pgo-lto) # PGO + LTO builds
+        OS="linuxpgolto"
+        ;;
     *-musl)
         OS="musl"
         ;;