Merge branch 'main' into no-gpl

.buildkite/hooks/post-checkout

@@ -22,6 +22,9 @@ if [[ "${DISABLE_UPSTREAM_CACHE}" != "true" ]]; then
         mkdir -p "$(dirname "${UPSTREAM_CACHE}")"
         git clone --mirror "${UPSTREAM_URL}" "${UPSTREAM_CACHE}"
     else
+        echo "Prune cache for ${UPSTREAM_URL}"
+        rm -f "${UPSTREAM_CACHE}/gc.log"
+        git -C "${UPSTREAM_CACHE}" prune
         echo "Update cache for ${UPSTREAM_URL}"
         git -C "${UPSTREAM_CACHE}" fetch
     fi
@@ -35,7 +38,16 @@ rm -rf ..?* .[!.]* *
 UPSTREAM_GITREF="$(buildkite-agent meta-data get --default "origin/${UPSTREAM_BRANCH}" BUILDKITE_JULIA_VERSION)"
 
 if [[ "${DISABLE_UPSTREAM_CACHE}" != "true" ]]; then
-    git clone --dissociate --reference "${UPSTREAM_CACHE}" "${UPSTREAM_URL}" "./"
+    # For some unknown reason, the `--dissociate` flag (which causes a `git repack -a -d`
+    # to be run after the clone is set) causes a spurious error on macOS:
+    #     fatal: unable to parse commit 1a2b3c3d
+    #     warning: Clone succeeded, but checkout failed.
+    #     You can inspect what was checked out with 'git status'
+    #     and retry with 'git restore --source=HEAD :/'
+    # This occurs when cloning manually as well, however the `git reset --hard`
+    # below tends to work.  So we just ignore the error with `|| true` here,
+    # and leave it to `git reset` to fail if there is actually a problem.
+    git clone --dissociate --reference "${UPSTREAM_CACHE}" "${UPSTREAM_URL}" "./" || true
 else
     git clone "${UPSTREAM_URL}" "./"
 fi

devdocs/macos-developer-id.md

@@ -0,0 +1,19 @@
+## How to see if the certificate is expired
+
+The main symptom will be that the upload job will be failing. The unlock keychain step prints the status of the certificate and it will print something like (CSSMERR_TP_CERT_EXPIRED) if the certificate is expired.
+
+## Update expired Apple Developer ID
+
+To replace the certificate you will need first a MacOS machine and to emit a new certificate.
+
+1. To replace an expired Apple certificate, clone julia-buildkite repo and clone https://github.com/staticfloat/cryptic-buildkite-plugin in its root.
+2. You can decrypt by running `make decrypt`.
+3. Get the macos_codesigning.keychain file in the `secrets` directory and add it to your local keychains with the Keychain Access app.
+4. From that app delete the old certificate and add the new one (it's a `.cer` file).
+5. Test the certificate by right clicking and running both the general evaluation and the codesigning one.
+6. Update the identity in `MACOS_CODESIGN_IDENTITY` (You can find the identity by doing `security find-identity -p codesigning $(PATH_TO_KEYCHAIN)/macos_codesigning.keychain` ).
+7. You can also test it by running the codesign.sh script in this repo with `./utilities/macos/codesign.sh --keychain ./secrets/macos_codesigning.keychain --identity $(NEW_IDENTITY) ./test` with some executable.
+8. Afterward reencrypt the keychain by running `./cryptic-buildkite-plugin/bin/encrypt_file --private-key=$(INSERT_PRIVATE_KEY) --repo-key=$(INSERT_REPO_KEY) ./secrets/macos_codesigning.keychain`
+9. Finally sign the repo with `make sign_treehashes`
+
+The `security` cli app is also useful for debugging and managing keychains. You can find more information about it by running `man security` in the terminal.

pipelines/main/launch_signed_jobs.yml.signature

@@ -1,3 +1 @@
-Salted__��,�`jsl@�dڇ��^��A{���8%�)��Y
-t�2��t�`�}B�ȮriD�ą�* U�����2S��\�
-�aB
Se��H�ss!
+Salted__m{KRF�{7y��-u�Ʉ�6����a�OU)4U�[����@3R�d�+��?D���ɴPU���GϦ%��9��!+��F�=��

pipelines/main/launch_unsigned_jobs.yml

@@ -102,6 +102,11 @@ steps:
               bash .buildkite/utilities/arches_pipeline_upload.sh \
               .buildkite/pipelines/main/platforms/test_macos.arches \
               .buildkite/pipelines/main/platforms/test_macos.yml
+          # Launch FreeBSD test jobs
+          GROUP="Test" \
+              bash .buildkite/utilities/arches_pipeline_upload.sh \
+              .buildkite/pipelines/main/platforms/test_freebsd.arches \
+              .buildkite/pipelines/main/platforms/test_freebsd.yml
           # Launch windows test jobs
           GROUP="Test" \
               bash .buildkite/utilities/arches_pipeline_upload.sh \

pipelines/main/launch_upload_jobs.yml.signature

@@ -1 +1 @@
-Salted__�xR���R�7餖�=����>S�Y`�j�Wr'�&��*VM�D����u-b��ɖ?+yϭ�#��ɑcZP��p�[4vE�u��
+Salted__�>a���{䮆s�T�U�>0Q+��V�/�^������Ɉe�<t���H�x¨*�3:�`��A:��W��	N����Lq�a�E>

pipelines/main/platforms/build_windows.arches

@@ -1,6 +1,6 @@
 # OS     TRIPLET              ARCH       DOCKER_ARCH   MAKE_FLAGS                                TIMEOUT     DOCKER_TAG
-windows  x86_64-w64-mingw32   x86_64     x86_64        VERBOSE=1                                 .           v7.2
-windows  i686-w64-mingw32     x86_64     i686          VERBOSE=1                                 .           v7.2
+windows  x86_64-w64-mingw32   x86_64     x86_64        VERBOSE=1                                 .           v7.10
+windows  i686-w64-mingw32     x86_64     i686          VERBOSE=1                                 .           v7.10
 
 # These special lines allow us to embed default values for the columns above.
 # Any column without a default mapping here will simply substitute a `.` to the empty string

pipelines/main/platforms/test_freebsd.arches

@@ -1,4 +1,5 @@
 # OS       TRIPLET                 ARCH          TIMEOUT
+freebsd    x86_64-unknown-freebsd  x86_64        .
 
 # These special lines allow us to embed default values for the columns above.
 # Any column without a default mapping here will simply substitute a `.` to the empty string

pipelines/main/platforms/test_freebsd.soft_fail.arches

@@ -1,5 +1,4 @@
 # OS       TRIPLET                 ARCH          TIMEOUT
-freebsd    x86_64-unknown-freebsd  x86_64        .
 
 # These special lines allow us to embed default values for the columns above.
 # Any column without a default mapping here will simply substitute a `.` to the empty string

pipelines/main/platforms/upload_windows.arches

@@ -1,6 +1,6 @@
 # OS     TRIPLET                DOCKER_TAG   TIMEOUT
-windows  x86_64-w64-mingw32     v5.44        .
-windows  i686-w64-mingw32       v5.44        .
+windows  x86_64-w64-mingw32     v7.10        .
+windows  i686-w64-mingw32       v7.10        .
 
 # These special lines allow us to embed default values for the columns above.
 # Any column without a default mapping here will simply substitute a `.` to the empty string

pipelines/main/platforms/upload_windows.yml

@@ -21,7 +21,9 @@ steps:
               variables:
                 - AWS_ACCESS_KEY_ID="U2FsdGVkX184v87+NPs3j9r/JoIuOrYt4/Z4wnRdklnY17NP8C8AMZvWYLJfT9t1"
                 - AWS_SECRET_ACCESS_KEY="U2FsdGVkX1+qptnxR/Mo5jZdH8OQfflRPiQBEhjgZIiTpn8KNCJYh/Cb8xxaUWazlcM9ceOlo0InDubL+J8zdg=="
-                - WINDOWS_CODESIGN_PASSWORD="U2FsdGVkX1+wiqniliFf7YWn1f/Y2rppITsOMvLGgK8n+GYWXWJH+POojLhEwU75"
+                - AZURE_TENANT_ID="U2FsdGVkX1+c2HbeSTq1c0fEep5riO3+PSQCiovdRT5akVpyy5hC311cXUCUB+A7ivIezAdJ7oz3dt19t60iuw=="
+                - AZURE_CLIENT_ID="U2FsdGVkX195QeT0ywNdXSa2ctMb9DFXyFVo0zG0/GtZUw95PTycqSZ+qEwXZbRZhGxXjGMVzcP2XZ0f/OuNnQ=="
+                - AZURE_CLIENT_SECRET="U2FsdGVkX1/905nz82Iy5toukRXEO8QS2etwfFCocofgnpBB62wJRDrWClPCNEX6ICo0jQzgoSbwDr2+C1ETSA=="
               files:
                 - .buildkite/secrets/tarball_signing.gpg
                 - .buildkite/secrets/windows_codesigning.pfx
@@ -42,9 +44,11 @@ steps:
                 # Have to include this for `buildkite-agent` to work:
                 - "BUILDKITE_AGENT_ACCESS_TOKEN"
                 # Have to include these for codesigning and uploading
-                - "WINDOWS_CODESIGN_PASSWORD"
                 - "AWS_ACCESS_KEY_ID"
                 - "AWS_SECRET_ACCESS_KEY"
+                - "AZURE_TENANT_ID"
+                - "AZURE_CLIENT_ID"
+                - "AZURE_CLIENT_SECRET"
                 # These control where we upload, important when running on julia-buildkite
                 - "S3_BUCKET"
                 - "S3_BUCKET_PREFIX"

pipelines/scheduled/launch_upload_jobs.yml.signature

@@ -1 +1,3 @@
-Salted__*b��]��#�At����t�k�ă��ڟ��W�EE�j�E�/YV�Yl"�T`�����&8�]�s�YG�I@	^]�0���E�ys
+Salted__��C;7X�G��D����B
+�� �r{�bP�Ш�|�N ����c�ɰ�ۑ
+����"!�0CC7�����g�J(�T��v���

utilities/build_julia.sh

@@ -38,7 +38,19 @@ MFLAGS+=( "-j${JULIA_CPU_THREADS}")
 
 # Add a few default flags to our make flags:
 MFLAGS+=( "VERBOSE=1" )
-MFLAGS+=( "TAGGED_RELEASE_BANNER=Official https://julialang.org/ release" )
+# Taken from https://stackoverflow.com/a/4024263
+verlte() {
+    printf '%s\n' "$1" "$2" | sort -C -V
+}
+verlt() {
+    ! verlte "$2" "$1"
+}
+if verlt "1.12" "$(cat VERSION)"; then
+    MFLAGS+=( "TAGGED_RELEASE_BANNER=Official https://julialang.org release" )
+else
+    # Keep trailing slash for compatability. The slash was removed in 1.12 with https://github.com/JuliaLang/julia/pull/53978
+    MFLAGS+=( "TAGGED_RELEASE_BANNER=Official https://julialang.org/ release" )
+fi
 MFLAGS+=( "JULIA_CPU_TARGET=${JULIA_CPU_TARGET}" )
 
 # Finish off with any extra make flags from the `.arches` file

utilities/test_julia.sh

@@ -33,7 +33,7 @@ if [[ "${OS}" == "macos" ]]; then
     echo "--- [mac] Codesigning"
     .buildkite/utilities/macos/codesign.sh "${JULIA_INSTALL_DIR}"
     echo "--- [mac] Update checksums for stdlib cachefiles after codesigning"
-    ${JULIA_INSTALL_DIR}/bin/julia .buildkite/utilities/update_stdlib_pkgimage_checksums.jl
+    JULIA_DEBUG=all ${JULIA_INSTALL_DIR}/bin/julia .buildkite/utilities/update_stdlib_pkgimage_checksums.jl
 fi
 
 

utilities/update_stdlib_pkgimage_checksums.jl

@@ -99,4 +99,4 @@ for dir in readdir(stdlib_cache_dir, join = true)
 end
 
 # check stdlibs caches are valid
-Base.isprecompiled(Base.PkgId(Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) || error()
+Base.isprecompiled(Base.PkgId(Base.UUID("8bb1440f-4735-579b-a4ab-409b98df4dab"), "DelimitedFiles")) || error()

utilities/upload_julia.sh

@@ -24,7 +24,7 @@ if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then
 
         # This _must_ be an absolute path
         KEYCHAIN_PATH="$(pwd)/.buildkite/secrets/macos_codesigning.keychain"
-        MACOS_CODESIGN_IDENTITY="2053E9292809B66582CA9F042B470C0929340362"
+        MACOS_CODESIGN_IDENTITY="E7CEA0DEF3BD5B83E9C50D9318845219097F43A0"
 
         # Add the keychain to the list of keychains to search, then unlock it
         security -v list-keychains -s -d user "${KEYCHAIN_PATH}"
@@ -81,15 +81,15 @@ if [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then
             /F"${UPLOAD_FILENAME}" \
             /O"$(cygpath -w "$(pwd)")" \
             /Dsign=true \
-            /Smysigntool="bash.exe '${codesign_script}' --certificate='${certificate}' \$f" \
+            /Smysigntool="bash.exe '${codesign_script}' \$f" \
             "$(cygpath -w "${iss_file}")"
 
         # Add the `.exe` to our upload targets
         UPLOAD_EXTENSIONS+=( "exe" )
 
         # Next, directly codesign every executable file in the install dir
         echo "--- [windows] Codesign everything in the install directory"
-        "${codesign_script}" --certificate="${certificate}" "${JULIA_INSTALL_DIR}"
+        "${codesign_script}" "${JULIA_INSTALL_DIR}"
 
         echo "--- [windows] Update checksums for stdlib cachefiles"
         ${JULIA_INSTALL_DIR}/bin/julia .buildkite/utilities/update_stdlib_pkgimage_checksums.jl

utilities/windows/codesign.sh

@@ -3,14 +3,18 @@
 
 set -euo pipefail
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+DLIB_DEFAULT_PATH='C:\Program Files\TrustedSigning\bin\x64\Azure.CodeSigning.Dlib.dll'
+DLIB_PATH="${DLIB_DEFAULT_PATH}"
+METADATA_JSON_PATH="$(cygpath -w ${SCRIPT_DIR}/codesign_metadata.json)"
+
 usage() {
-    echo "Usage: $0 --certificate=<path> --password=<password> <target>"
+    echo "Usage: $0 [--dlib-path=<path>] <target>"
     echo
     echo "Parameter descriptions:"
     echo
-    echo "       key: A '.pfx' file that contains the codesigning certificate"
-    echo
-    echo "  password: The password to unlock the given '.pfx' file."
+    echo "  dlib-path: The path to the Trusted Signing .dlib file (defaults to ${DLIB_DEFAULT_PATH})."
     echo
     echo "    target: A file or directory to codesign (must come last!)"
 }
@@ -26,22 +30,13 @@ fi
 
 while [ "$#" -gt 1 ]; do
     case "${1}" in
-        --certificate)
-            CERT_PATH="$2"
+        --dlib-path)
+            DLIB_PATH="$2"
             shift
             shift
             ;;
-        --certificate=*)
-            CERT_PATH="${1#*=}"
-            shift
-            ;;
-        --password)
-            CERT_PASSWORD="$2"
-            shift
-            shift
-            ;;
-        --password=*)
-            CERT_PASSWORD="${1#*=}"
+        --dlib-path=*)
+            DLIB_PATH="${1#*=}"
             shift
             ;;
         *)
@@ -51,32 +46,33 @@ while [ "$#" -gt 1 ]; do
             ;;
     esac
 done
+DLIB_PATH="$(cygpath -w "${DLIB_PATH}")"
 
-# We tend to receive this via an environment variable on CI, so as to
-# not print it out when `make` is run in verbose mode
-CERT_PASSWORD="${CERT_PASSWORD:-${WINDOWS_CODESIGN_PASSWORD}}"
+if [[ -z "${AZURE_TENANT_ID:-}" ]] ||
+   [[ -z "${AZURE_CLIENT_ID:-}" ]] ||
+   [[ -z "${AZURE_CLIENT_SECRET:-}" ]]; then
+    echo "ERROR: Missing AZURE_* secret variables!" >&2
+    exit 1
+fi
 
-if [[ ! -f "${CERT_PATH}" ]]; then
-    echo "ERROR: Certificate path '${CERT_PATH}' does not exist!" >&2
+if [[ ! -f "${DLIB_PATH}" ]]; then
+    echo "ERROR: No Trusted Signing dlib found at '${DLIB_PATH}'" >&2
     exit 1
 fi
-CERT_PATH="$(cygpath -w "$(abspath "${CERT_PATH}")")"
+
 
 # We will try to codesign, using multiple timestamping servers in case one is down
 SERVERS=(
-    "http://timestamp.digicert.com/?alg=sha1"
-    "http://timestamp.globalsign.com/scripts/timstamp.dll"
-    "http://timestamp.comodoca.com/authenticode"
+    "http://timestamp.acs.microsoft.com"
+    "http://timestamp.digicert.com"
     "http://tsa.starfieldtech.com"
 )
 NUM_RETRIES=3
 
 function do_codesign() {
     for retry in $(seq 1 ${NUM_RETRIES}); do
         for SERVER in ${SERVERS[@]}; do
-            # Note that we're using SHA1 signing here, because that's what our certificate supports.
-            # In the future, we may be able to upgrade to SHA256.
-            if MSYS2_ARG_CONV_EXCL='*' signtool sign /debug /fd certHash /f "${CERT_PATH}" /p "${CERT_PASSWORD}" /t "${SERVER}" "$1"; then
+            if MSYS2_ARG_CONV_EXCL='*' signtool sign /q /fd SHA256 /tr "${SERVER}" /td SHA256 /dlib "${DLIB_PATH}" /dmdf "${METADATA_JSON_PATH}" "$1"; then
                 return 0
             fi
         done
@@ -89,7 +85,7 @@ function do_codesign() {
 # This codesign script only works on files
 if [ -f "${1}" ]; then
     # If we're codesigning a single file, directly invoke codesign on that file
-    echo "Codesigning file ${1} with certificate ${CERT_PATH}"
+    echo "Codesigning file ${1}"
     do_codesign "${1}"
 elif [ -d "${1}" ]; then
     # Create a fifo to communicate from `find` to `while`
@@ -105,7 +101,7 @@ elif [ -d "${1}" ]; then
     # This while loop reads in from the fifo, and invokes `do_codesign`,
     # but it does so in a background task, so that the codesigning can
     # happen in parallel.  This speeds things up by a few seconds.
-    echo "Codesigning dir ${1} with certificate ${CERT_PATH}"
+    echo "Codesigning dir ${1}"
     NUM_CODESIGNS=0
     while IFS= read -r -d '' exe_file; do
         do_codesign "${exe_file}" &

utilities/windows/codesign_metadata.json

@@ -0,0 +1,16 @@
+{
+    "Endpoint": "https://eus.codesigning.azure.net/",
+    "CodeSigningAccountName": "juliahubwincertsaccount",
+    "CertificateProfileName": "JuliaHubWinCert",
+    "ExcludeCredentials": [
+        "WorkloadIdentityCredential",
+        "ManagedIdentityCredential",
+        "SharedTokenCacheCredential",
+        "VisualStudioCredential",
+        "VisualStudioCodeCredential",
+        "AzureCliCredential",
+        "AzurePowerShellCredential",
+        "AzureDeveloperCliCredential",
+        "InteractiveBrowserCredential"
+    ]
+}