Merge bitcoin/bitcoin#31221: ci: Split out native fuzz jobs for macOS… #1643
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2023 The Bitcoin Core developers | |
| # Distributed under the MIT software license, see the accompanying | |
| # file COPYING or http://www.opensource.org/licenses/mit-license.php. | |
| name: CI | |
| on: | |
| # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request. | |
| pull_request: | |
| # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push. | |
| push: | |
| branches: | |
| - '**' | |
| tags-ignore: | |
| - '**' | |
| concurrency: | |
| group: ${{ github.event_name != 'pull_request' && github.run_id || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CI_FAILFAST_TEST_LEAVE_DANGLING: 1 # GHA does not care about dangling processes and setting this variable avoids killing the CI script itself on error | |
| MAKEJOBS: '-j10' | |
| jobs: | |
| test-each-commit: | |
| name: 'test each commit' | |
| runs-on: ubuntu-24.04 | |
| if: github.event_name == 'pull_request' && github.event.pull_request.commits != 1 | |
| timeout-minutes: 360 # Use maximum time, see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes. Assuming a worst case time of 1 hour per commit, this leads to a --max-count=6 below. | |
| env: | |
| MAX_COUNT: 6 | |
| steps: | |
| - name: Determine fetch depth | |
| run: echo "FETCH_DEPTH=$((${{ github.event.pull_request.commits }} + 2))" >> "$GITHUB_ENV" | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: ${{ env.FETCH_DEPTH }} | |
| - name: Determine commit range | |
| run: | | |
| # Checkout HEAD~ and find the test base commit | |
| # Checkout HEAD~ because it would be wasteful to rerun tests on the PR | |
| # head commit that are already run by other jobs. | |
| git checkout HEAD~ | |
| # Figure out test base commit by listing ancestors of HEAD, excluding | |
| # ancestors of the most recent merge commit, limiting the list to the | |
| # newest MAX_COUNT ancestors, ordering it from oldest to newest, and | |
| # taking the first one. | |
| # | |
| # If the branch contains up to MAX_COUNT ancestor commits after the | |
| # most recent merge commit, all of those commits will be tested. If it | |
| # contains more, only the most recent MAX_COUNT commits will be | |
| # tested. | |
| # | |
| # In the command below, the ^@ suffix is used to refer to all parents | |
| # of the merge commit as described in: | |
| # https://git-scm.com/docs/git-rev-parse#_other_rev_parent_shorthand_notations | |
| # and the ^ prefix is used to exclude these parents and all their | |
| # ancestors from the rev-list output as described in: | |
| # https://git-scm.com/docs/git-rev-list | |
| MERGE_BASE=$(git rev-list -n1 --merges HEAD) | |
| EXCLUDE_MERGE_BASE_ANCESTORS= | |
| # MERGE_BASE can be empty due to limited fetch-depth | |
| if test -n "$MERGE_BASE"; then | |
| EXCLUDE_MERGE_BASE_ANCESTORS=^${MERGE_BASE}^@ | |
| fi | |
| echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD $EXCLUDE_MERGE_BASE_ANCESTORS | head -1)" >> "$GITHUB_ENV" | |
| - run: | | |
| sudo apt-get update | |
| sudo apt-get install clang ccache build-essential cmake pkg-config python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libzmq3-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y | |
| - name: Compile and run tests | |
| run: | | |
| # Run tests on commits after the last merge commit and before the PR head commit | |
| # Use clang++, because it is a bit faster and uses less memory than g++ | |
| git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DWERROR=ON -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWITH_BDB=ON -DWITH_USDT=ON -DCMAKE_CXX_FLAGS='-Wno-error=unused-member-function' && cmake --build build -j $(nproc) && ctest --output-on-failure --stop-on-failure --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 ))" ${{ env.TEST_BASE }} | |
| macos-native-arm64: | |
| name: ${{ matrix.job-name }} | |
| # Use latest image, but hardcode version to avoid silent upgrades (and breaks). | |
| # See: https://github.com/actions/runner-images#available-images. | |
| runs-on: macos-14 | |
| # When a contributor maintains a fork of the repo, any pull request they make | |
| # to their own fork, or to the main repository, will trigger two CI runs: | |
| # one for the branch push and one for the pull request. | |
| # This can be avoided by setting SKIP_BRANCH_PUSH=true as a custom env variable | |
| # in Github repository settings. | |
| if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} | |
| timeout-minutes: 120 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job-type: [standard, fuzz] | |
| include: | |
| - job-type: standard | |
| file-env: './ci/test/00_setup_env_mac_native.sh' | |
| job-name: 'macOS 14 native, arm64, no depends, sqlite only, gui' | |
| - job-type: fuzz | |
| file-env: './ci/test/00_setup_env_mac_native_fuzz.sh' | |
| job-name: 'macOS 14 native, arm64, fuzz' | |
| env: | |
| DANGER_RUN_CI_ON_HOST: 1 | |
| BASE_ROOT_DIR: ${{ github.workspace }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Clang version | |
| run: | | |
| sudo xcode-select --switch /Applications/Xcode_15.0.app | |
| clang --version | |
| - name: Install Homebrew packages | |
| env: | |
| HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 | |
| run: | | |
| # A workaround for "The `brew link` step did not complete successfully" error. | |
| brew install --quiet python@3 || brew link --overwrite python@3 | |
| # Update to install pkgconf, once the GHA image has been updated. | |
| brew install --quiet coreutils ninja gnu-getopt ccache boost libevent zeromq qt@5 qrencode | |
| - name: Set Ccache directory | |
| run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" | |
| - name: Restore Ccache cache | |
| id: ccache-cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: ${{ github.job }}-${{ matrix.job-type }}-ccache-${{ github.run_id }} | |
| restore-keys: ${{ github.job }}-${{ matrix.job-type }}-ccache- | |
| - name: CI script | |
| run: ./ci/test_run_all.sh | |
| env: | |
| FILE_ENV: ${{ matrix.file-env }} | |
| - name: Save Ccache cache | |
| uses: actions/cache/save@v4 | |
| if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true' | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache | |
| key: ${{ github.job }}-${{ matrix.job-type }}-ccache-${{ github.run_id }} | |
| win64-native: | |
| name: ${{ matrix.job-name }} | |
| # Use latest image, but hardcode version to avoid silent upgrades (and breaks). | |
| # See: https://github.com/actions/runner-images#available-images. | |
| runs-on: windows-2022 | |
| if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} | |
| env: | |
| PYTHONUTF8: 1 | |
| TEST_RUNNER_TIMEOUT_FACTOR: 40 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job-type: [standard, fuzz] | |
| include: | |
| - job-type: standard | |
| generate-options: '-DBUILD_GUI=ON -DWITH_BDB=ON -DWITH_ZMQ=ON -DBUILD_BENCH=ON -DWERROR=ON' | |
| job-name: 'Win64 native, VS 2022' | |
| - job-type: fuzz | |
| generate-options: '-DVCPKG_MANIFEST_NO_DEFAULT_FEATURES=ON -DVCPKG_MANIFEST_FEATURES="sqlite" -DBUILD_GUI=OFF -DBUILD_FOR_FUZZING=ON -DWERROR=ON' | |
| job-name: 'Win64 native fuzz, VS 2022' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Configure Developer Command Prompt for Microsoft Visual C++ | |
| # Using microsoft/setup-msbuild is not enough. | |
| uses: ilammy/msvc-dev-cmd@v1 | |
| with: | |
| arch: x64 | |
| - name: Get tool information | |
| run: | | |
| cmake -version | Tee-Object -FilePath "cmake_version" | |
| Write-Output "---" | |
| msbuild -version | Tee-Object -FilePath "msbuild_version" | |
| $env:VCToolsVersion | Tee-Object -FilePath "toolset_version" | |
| py -3 --version | |
| Write-Host "PowerShell version $($PSVersionTable.PSVersion.ToString())" | |
| - name: Using vcpkg with MSBuild | |
| run: | | |
| Set-Location "$env:VCPKG_INSTALLATION_ROOT" | |
| Add-Content -Path "triplets\x64-windows.cmake" -Value "set(VCPKG_BUILD_TYPE release)" | |
| Add-Content -Path "triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)" | |
| - name: vcpkg tools cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: C:/vcpkg/downloads/tools | |
| key: ${{ github.job }}-vcpkg-tools | |
| - name: Restore vcpkg binary cache | |
| uses: actions/cache/restore@v4 | |
| id: vcpkg-binary-cache | |
| with: | |
| path: ~/AppData/Local/vcpkg/archives | |
| key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} | |
| - name: Generate build system | |
| run: | | |
| cmake -B build --preset vs2022-static -DCMAKE_TOOLCHAIN_FILE="$env:VCPKG_INSTALLATION_ROOT\scripts\buildsystems\vcpkg.cmake" ${{ matrix.generate-options }} | |
| - name: Save vcpkg binary cache | |
| uses: actions/cache/save@v4 | |
| if: github.event_name != 'pull_request' && steps.vcpkg-binary-cache.outputs.cache-hit != 'true' && matrix.job-type == 'standard' | |
| with: | |
| path: ~/AppData/Local/vcpkg/archives | |
| key: ${{ github.job }}-vcpkg-binary-${{ hashFiles('cmake_version', 'msbuild_version', 'toolset_version', 'vcpkg.json') }} | |
| - name: Build | |
| working-directory: build | |
| run: | | |
| cmake --build . -j $env:NUMBER_OF_PROCESSORS --config Release | |
| - name: Run test suite | |
| if: matrix.job-type == 'standard' | |
| working-directory: build | |
| run: | | |
| ctest --output-on-failure --stop-on-failure -j $env:NUMBER_OF_PROCESSORS -C Release | |
| - name: Run functional tests | |
| if: matrix.job-type == 'standard' | |
| working-directory: build | |
| env: | |
| BITCOIND: '${{ github.workspace }}\build\src\Release\bitcoind.exe' | |
| BITCOINCLI: '${{ github.workspace }}\build\src\Release\bitcoin-cli.exe' | |
| BITCOINUTIL: '${{ github.workspace }}\build\src\Release\bitcoin-util.exe' | |
| BITCOINWALLET: '${{ github.workspace }}\build\src\Release\bitcoin-wallet.exe' | |
| TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }} | |
| shell: cmd | |
| run: py -3 test\functional\test_runner.py --jobs %NUMBER_OF_PROCESSORS% --ci --quiet --tmpdirprefix=%RUNNER_TEMP% --combinedlogslen=99999999 --timeout-factor=%TEST_RUNNER_TIMEOUT_FACTOR% %TEST_RUNNER_EXTRA% | |
| - name: Clone corpora | |
| if: matrix.job-type == 'fuzz' | |
| run: | | |
| git clone --depth=1 https://github.com/bitcoin-core/qa-assets "$env:RUNNER_TEMP\qa-assets" | |
| Set-Location "$env:RUNNER_TEMP\qa-assets" | |
| Write-Host "Using qa-assets repo from commit ..." | |
| git log -1 | |
| - name: Run fuzz tests | |
| if: matrix.job-type == 'fuzz' | |
| working-directory: build | |
| env: | |
| BITCOINFUZZ: '${{ github.workspace }}\build\src\test\fuzz\Release\fuzz.exe' | |
| shell: cmd | |
| run: | | |
| py -3 test\fuzz\test_runner.py --par %NUMBER_OF_PROCESSORS% --loglevel DEBUG %RUNNER_TEMP%\qa-assets\fuzz_corpora | |
| asan-lsan-ubsan-integer-no-depends-usdt: | |
| name: 'ASan + LSan + UBSan + integer, no depends, USDT' | |
| runs-on: ubuntu-24.04 # has to match container in ci/test/00_setup_env_native_asan.sh for tracing tools | |
| if: ${{ vars.SKIP_BRANCH_PUSH != 'true' || github.event_name == 'pull_request' }} | |
| timeout-minutes: 120 | |
| env: | |
| FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" | |
| DANGER_CI_ON_HOST_CACHE_FOLDERS: 1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set Ccache directory | |
| run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" | |
| - name: Set base root directory | |
| run: echo "BASE_ROOT_DIR=${RUNNER_TEMP}" >> "$GITHUB_ENV" | |
| - name: Restore Ccache cache | |
| id: ccache-cache | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| key: ${{ github.job }}-ccache-${{ github.run_id }} | |
| restore-keys: ${{ github.job }}-ccache- | |
| - name: Enable bpfcc script | |
| # In the image build step, no external environment variables are available, | |
| # so any settings will need to be written to the settings env file: | |
| run: sed -i "s|\${INSTALL_BCC_TRACING_TOOLS}|true|g" ./ci/test/00_setup_env_native_asan.sh | |
| - name: CI script | |
| run: ./ci/test_run_all.sh | |
| - name: Save Ccache cache | |
| uses: actions/cache/save@v4 | |
| if: github.event_name != 'pull_request' && steps.ccache-cache.outputs.cache-hit != 'true' | |
| with: | |
| path: ${{ env.CCACHE_DIR }} | |
| # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache | |
| key: ${{ github.job }}-ccache-${{ github.run_id }} |