From 5319ee0fa96faaabb00690c4b78e98c15f6943aa Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:52:43 +0000 Subject: [PATCH 01/15] merge bitcoin#27314: Fix handling of `CXX=clang++` when building `qt` package --- depends/packages/qt.mk | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 2333cc883107b..6cb6e53ed74e5 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -159,9 +159,15 @@ $(package)_config_opts_linux += -dbus-runtime ifneq ($(LTO),) $(package)_config_opts_linux += -ltcg endif -$(package)_config_opts_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ -ifneq (,$(findstring -stdlib=libc++,$($(1)_cxx))) -$(package)_config_opts_x86_64_linux = -xplatform linux-clang-libc++ + +ifneq (,$(findstring clang,$($(package)_cxx))) + ifneq (,$(findstring -stdlib=libc++,$($(package)_cxx))) + $(package)_config_opts_linux += -platform linux-clang-libc++ -xplatform linux-clang-libc++ + else + $(package)_config_opts_linux += -platform linux-clang -xplatform linux-clang + endif +else + $(package)_config_opts_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ endif $(package)_config_opts_mingw32 = -no-opengl From 3fe6d247b53b11a8d7a90ed0dbd38d37806d0ea4 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:14:08 +0000 Subject: [PATCH 02/15] merge bitcoin#28954: Reduce use of bash -c --- ci/dash/test_integrationtests.sh | 2 +- ci/dash/test_unittests.sh | 4 ++-- ci/test/00_setup_env.sh | 1 - ci/test/00_setup_env_native_multiprocess.sh | 2 +- ci/test/00_setup_env_s390x.sh | 1 - 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ci/dash/test_integrationtests.sh b/ci/dash/test_integrationtests.sh index 8a0013b1a94d8..86a00c8f1b0ce 100755 --- a/ci/dash/test_integrationtests.sh +++ b/ci/dash/test_integrationtests.sh @@ -42,7 +42,7 @@ echo "Using socketevents mode: $SOCKETEVENTS" EXTRA_ARGS="--dashd-arg=-socketevents=$SOCKETEVENTS" set +e -LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS +LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS RESULT=$? set -e diff --git a/ci/dash/test_unittests.sh b/ci/dash/test_unittests.sh index b3212327107fd..37a0a4575e079 100755 --- a/ci/dash/test_unittests.sh +++ b/ci/dash/test_unittests.sh @@ -29,8 +29,8 @@ if [ "$DIRECT_WINE_EXEC_TESTS" = "true" ]; then wine ./src/test/test_dash.exe else if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - ${TEST_RUNNER_ENV} ./src/test/test_dash --catch_system_errors=no -l test_suite + ./src/test/test_dash --catch_system_errors=no -l test_suite else - ${TEST_RUNNER_ENV} make $MAKEJOBS check VERBOSE=1 + make $MAKEJOBS check VERBOSE=1 fi fi diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 587dfc2bfefa6..2023705cf8163 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -44,7 +44,6 @@ export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} # This is needed because some ci machines have slow CPU or disk, so sanitizers # might be slow or a reindex might be waiting on disk IO. export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-4} -export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index c070da4ef0786..d6727d05b0611 100755 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -11,4 +11,4 @@ export PACKAGES="cmake python3 llvm clang" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" export BITCOIN_CONFIG="--with-boost-process --enable-debug CC=clang CXX=clang++" # Use clang to avoid OOM -export TEST_RUNNER_ENV="BITCOIND=dash-node" +export BITCOIND=dash-node # Used in functional tests diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index c71a4891b9f19..a46bde95b5584 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -19,7 +19,6 @@ fi # Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x export RUN_UNIT_TESTS=true -export TEST_RUNNER_ENV="LC_ALL=C" export RUN_FUNCTIONAL_TESTS=true export GOAL="install" export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --with-boost-process" # GUI tests disabled for now, see https://github.com/bitcoin/bitcoin/issues/23730 From 5c36005ea8ac9e07f54a6f6810371a2ce08cb426 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:15:23 +0000 Subject: [PATCH 03/15] trivial: sort `BUILD_TARGET` on GitHub and in `matrix.sh` alphabetically --- .github/workflows/build.yml | 12 ++++++------ ci/dash/matrix.sh | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b9dbd80207f2..ebccbcec91aae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,22 +114,22 @@ jobs: - build_target: linux64 host: x86_64-pc-linux-gnu depends_on: linux64 - - build_target: linux64_tsan - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_ubsan + - build_target: linux64_cxx20 host: x86_64-pc-linux-gnu depends_on: linux64 - build_target: linux64_fuzz host: x86_64-pc-linux-gnu depends_on: linux64 - - build_target: linux64_cxx20 + - build_target: linux64_nowallet host: x86_64-pc-linux-gnu depends_on: linux64 - build_target: linux64_sqlite host: x86_64-pc-linux-gnu depends_on: linux64 - - build_target: linux64_nowallet + - build_target: linux64_tsan + host: x86_64-pc-linux-gnu + depends_on: linux64 + - build_target: linux64_ubsan host: x86_64-pc-linux-gnu depends_on: linux64 container: diff --git a/ci/dash/matrix.sh b/ci/dash/matrix.sh index 82a9233df17b9..6eeb2a7565317 100755 --- a/ci/dash/matrix.sh +++ b/ci/dash/matrix.sh @@ -18,28 +18,28 @@ export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ if [ "$BUILD_TARGET" = "arm-linux" ]; then source ./ci/test/00_setup_env_arm.sh -elif [ "$BUILD_TARGET" = "win64" ]; then - source ./ci/test/00_setup_env_win64.sh elif [ "$BUILD_TARGET" = "linux64" ]; then source ./ci/test/00_setup_env_native_qt5.sh elif [ "$BUILD_TARGET" = "linux64_asan" ]; then source ./ci/test/00_setup_env_native_asan.sh -elif [ "$BUILD_TARGET" = "linux64_tsan" ]; then - source ./ci/test/00_setup_env_native_tsan.sh -elif [ "$BUILD_TARGET" = "linux64_ubsan" ]; then - source ./ci/test/00_setup_env_native_ubsan.sh -elif [ "$BUILD_TARGET" = "linux64_fuzz" ]; then - source ./ci/test/00_setup_env_native_fuzz.sh elif [ "$BUILD_TARGET" = "linux64_cxx20" ]; then source ./ci/test/00_setup_env_native_cxx20.sh -elif [ "$BUILD_TARGET" = "linux64_sqlite" ]; then - source ./ci/test/00_setup_env_native_sqlite.sh +elif [ "$BUILD_TARGET" = "linux64_fuzz" ]; then + source ./ci/test/00_setup_env_native_fuzz.sh elif [ "$BUILD_TARGET" = "linux64_nowallet" ]; then source ./ci/test/00_setup_env_native_nowallet.sh +elif [ "$BUILD_TARGET" = "linux64_sqlite" ]; then + source ./ci/test/00_setup_env_native_sqlite.sh +elif [ "$BUILD_TARGET" = "linux64_tsan" ]; then + source ./ci/test/00_setup_env_native_tsan.sh +elif [ "$BUILD_TARGET" = "linux64_ubsan" ]; then + source ./ci/test/00_setup_env_native_ubsan.sh elif [ "$BUILD_TARGET" = "linux64_valgrind" ]; then source ./ci/test/00_setup_env_native_valgrind.sh elif [ "$BUILD_TARGET" = "mac" ]; then source ./ci/test/00_setup_env_mac.sh elif [ "$BUILD_TARGET" = "s390x" ]; then source ./ci/test/00_setup_env_s390x.sh +elif [ "$BUILD_TARGET" = "win64" ]; then + source ./ci/test/00_setup_env_win64.sh fi From 51762af1ba95c125fbed04bc6170c208dd2bf68d Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 19 Nov 2024 10:30:38 +0000 Subject: [PATCH 04/15] ci: use underscore to separate variant name from target triple --- .gitlab-ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6af1abbbe6382..15200ef9441ec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -189,13 +189,13 @@ x86_64-w64-mingw32: variables: HOST: x86_64-w64-mingw32 -x86_64-pc-linux-gnu-debug: +x86_64-pc-linux-gnu_debug: extends: .build-depends-template variables: HOST: x86_64-pc-linux-gnu DEP_OPTS: "DEBUG=1" -x86_64-pc-linux-gnu-nowallet: +x86_64-pc-linux-gnu_nowallet: extends: - .build-depends-template - .skip-in-fast-mode-template @@ -203,7 +203,7 @@ x86_64-pc-linux-gnu-nowallet: HOST: x86_64-pc-linux-gnu DEP_OPTS: "NO_WALLET=1" -x86_64-pc-linux-gnu-multiprocess: +x86_64-pc-linux-gnu_multiprocess: extends: - .build-depends-template - .skip-in-fast-mode-template @@ -239,7 +239,7 @@ win64-build: linux64-build: extends: .build-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_debug variables: BUILD_TARGET: linux64 @@ -248,7 +248,7 @@ linux64_cxx20-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_debug variables: BUILD_TARGET: linux64_cxx20 @@ -257,7 +257,7 @@ linux64_sqlite-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_debug variables: BUILD_TARGET: linux64_sqlite @@ -266,7 +266,7 @@ linux64_fuzz-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_debug variables: BUILD_TARGET: linux64_fuzz @@ -275,7 +275,7 @@ linux64_fuzz-build: # - .build-template # - .skip-in-fast-mode-template # needs: -# - x86_64-pc-linux-gnu-debug +# - x86_64-pc-linux-gnu_debug # variables: # BUILD_TARGET: linux64_asan @@ -284,7 +284,7 @@ linux64_tsan-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_debug variables: BUILD_TARGET: linux64_tsan @@ -293,7 +293,7 @@ linux64_ubsan-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_debug variables: BUILD_TARGET: linux64_ubsan @@ -302,7 +302,7 @@ linux64_nowallet-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-nowallet + - x86_64-pc-linux-gnu_nowallet variables: BUILD_TARGET: linux64_nowallet @@ -311,7 +311,7 @@ linux64_multiprocess-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-multiprocess + - x86_64-pc-linux-gnu_multiprocess variables: BUILD_TARGET: linux64_multiprocess @@ -320,7 +320,7 @@ linux64_multiprocess-build: # - .build-template # - .skip-in-fast-mode-template # needs: -# - x86_64-pc-linux-gnu-debug +# - x86_64-pc-linux-gnu_debug # variables: # BUILD_TARGET: linux64_valgrind From 881a21c49630ee28f7cdd538b84c5ad26b8ad99f Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 20 Nov 2024 04:29:53 +0000 Subject: [PATCH 05/15] ci: use GitLab depends naming convention on GitHub Actions runs --- .github/workflows/build.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ebccbcec91aae..65faa8ffca18b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,16 +56,16 @@ jobs: cache-to: type=inline build-depends: - name: Build Dependencies + name: ${{ matrix.depends_name }} needs: build-image runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: include: - - build_target: arm-linux + - depends_name: arm-linux-gnueabihf host: arm-linux-gnueabihf - - build_target: linux64 + - depends_name: x86_64-pc-linux-gnu host: x86_64-pc-linux-gnu container: @@ -92,16 +92,16 @@ jobs: path: | depends/built depends/${{ matrix.host }} - key: ${{ runner.os }}-depends-${{ matrix.build_target }}-${{ hashFiles('depends/packages/*') }} + key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }} restore-keys: | - ${{ runner.os }}-depends-${{ matrix.build_target }}-${{ hashFiles('depends/packages/*') }} - ${{ runner.os }}-depends-${{ matrix.build_target }} + ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }} + ${{ runner.os }}-depends-${{ matrix.depends_name }} - name: Build depends run: make -j$(nproc) -C depends HOST=${{ matrix.host }} build: - name: Build + name: ${{ matrix.build_target }} needs: [build-image, build-depends] runs-on: ubuntu-22.04 strategy: @@ -109,29 +109,29 @@ jobs: matrix: include: - build_target: arm-linux + depends_name: arm-linux-gnueabihf host: arm-linux-gnueabihf - depends_on: arm-linux - build_target: linux64 + depends_name: x86_64-pc-linux-gnu host: x86_64-pc-linux-gnu - depends_on: linux64 - build_target: linux64_cxx20 + depends_name: x86_64-pc-linux-gnu host: x86_64-pc-linux-gnu - depends_on: linux64 - build_target: linux64_fuzz + depends_name: x86_64-pc-linux-gnu host: x86_64-pc-linux-gnu - depends_on: linux64 - build_target: linux64_nowallet + depends_name: x86_64-pc-linux-gnu host: x86_64-pc-linux-gnu - depends_on: linux64 - build_target: linux64_sqlite + depends_name: x86_64-pc-linux-gnu host: x86_64-pc-linux-gnu - depends_on: linux64 - build_target: linux64_tsan + depends_name: x86_64-pc-linux-gnu host: x86_64-pc-linux-gnu - depends_on: linux64 - build_target: linux64_ubsan + depends_name: x86_64-pc-linux-gnu host: x86_64-pc-linux-gnu - depends_on: linux64 container: image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} options: --user root @@ -147,7 +147,7 @@ jobs: path: | depends/built depends/${{ matrix.host }} - key: ${{ runner.os }}-depends-${{ matrix.depends_on }}-${{ hashFiles('depends/packages/*') }} + key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }} - name: Determine PR Base SHA id: vars From 4ff23e570ee7794bc0935b4a666693bbc223690e Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:35:31 +0000 Subject: [PATCH 06/15] ci: use GitLab `DEP_OPTS` on GitHub Actions builds --- .github/workflows/build.yml | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 65faa8ffca18b..57b42308109aa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,8 +65,10 @@ jobs: include: - depends_name: arm-linux-gnueabihf host: arm-linux-gnueabihf - - depends_name: x86_64-pc-linux-gnu + dep_opts: "" + - depends_name: x86_64-pc-linux-gnu_debug host: x86_64-pc-linux-gnu + dep_opts: "DEBUG=1" container: image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} @@ -94,11 +96,10 @@ jobs: depends/${{ matrix.host }} key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }} restore-keys: | - ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }} - ${{ runner.os }}-depends-${{ matrix.depends_name }} + ${{ runner.os }}-depends-${{ matrix.depends_name }}- - name: Build depends - run: make -j$(nproc) -C depends HOST=${{ matrix.host }} + run: env HOST=${{ matrix.host }} ${{ matrix.dep_opts }} make -j$(nproc) -C depends build: name: ${{ matrix.build_target }} @@ -112,25 +113,25 @@ jobs: depends_name: arm-linux-gnueabihf host: arm-linux-gnueabihf - build_target: linux64 - depends_name: x86_64-pc-linux-gnu + depends_name: x86_64-pc-linux-gnu_debug host: x86_64-pc-linux-gnu - build_target: linux64_cxx20 - depends_name: x86_64-pc-linux-gnu + depends_name: x86_64-pc-linux-gnu_debug host: x86_64-pc-linux-gnu - build_target: linux64_fuzz - depends_name: x86_64-pc-linux-gnu + depends_name: x86_64-pc-linux-gnu_debug host: x86_64-pc-linux-gnu - build_target: linux64_nowallet - depends_name: x86_64-pc-linux-gnu + depends_name: x86_64-pc-linux-gnu_debug host: x86_64-pc-linux-gnu - build_target: linux64_sqlite - depends_name: x86_64-pc-linux-gnu + depends_name: x86_64-pc-linux-gnu_debug host: x86_64-pc-linux-gnu - build_target: linux64_tsan - depends_name: x86_64-pc-linux-gnu + depends_name: x86_64-pc-linux-gnu_debug host: x86_64-pc-linux-gnu - build_target: linux64_ubsan - depends_name: x86_64-pc-linux-gnu + depends_name: x86_64-pc-linux-gnu_debug host: x86_64-pc-linux-gnu container: image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} @@ -148,6 +149,8 @@ jobs: depends/built depends/${{ matrix.host }} key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }} + restore-keys: | + ${{ runner.os }}-depends-${{ matrix.depends_name }}- - name: Determine PR Base SHA id: vars From cb0ac9420959476159e0dd635b4a0da63db698ed Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 20 Nov 2024 13:58:32 +0000 Subject: [PATCH 07/15] ci: move matrix def. to `matrix.json`, add `DEP_OPTS` hash to cache key --- .github/workflows/build.yml | 70 ++++++++++++++++------------------- .github/workflows/matrix.json | 52 ++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/matrix.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57b42308109aa..7e39b112306f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,8 @@ jobs: outputs: image-tag: ${{ steps.prepare.outputs.image-tag }} repo-name: ${{ steps.prepare.outputs.repo-name }} + dep-matrix: ${{ steps.prepare.outputs.dep-matrix }} + src-matrix: ${{ steps.prepare.outputs.src-matrix }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -32,6 +34,8 @@ jobs: REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') echo "image-tag=${BRANCH_NAME}" >> $GITHUB_OUTPUT echo "repo-name=${REPO_NAME}" >> $GITHUB_OUTPUT + echo "dep-matrix=$(jq -r '.dep' -c .github/workflows/matrix.json)" >> $GITHUB_OUTPUT + echo "src-matrix=$(jq -r '.src' -c .github/workflows/matrix.json)" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -61,15 +65,7 @@ jobs: runs-on: ubuntu-22.04 strategy: fail-fast: false - matrix: - include: - - depends_name: arm-linux-gnueabihf - host: arm-linux-gnueabihf - dep_opts: "" - - depends_name: x86_64-pc-linux-gnu_debug - host: x86_64-pc-linux-gnu - dep_opts: "DEBUG=1" - + matrix: ${{ fromJson(needs.build-image.outputs.dep-matrix) }} container: image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} options: --user root @@ -88,14 +84,24 @@ jobs: restore-keys: | depends-sources- + - name: Determine params + id: det-params + run: | + dep_name="${{ matrix.depends_name }}" + dep_opts="${{ matrix.dep_opts }}" + dep_hash="$(echo -n ${dep_opts} | sha256sum | head -c 64)" + echo "\"${dep_name}\" has DEP_OPTS \"${dep_opts}\" with hash \"${dep_hash}\"" + echo "dep_hash=${dep_hash}" >> $GITHUB_OUTPUT + - name: Cache depends uses: actions/cache@v4 with: path: | depends/built depends/${{ matrix.host }} - key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }} + key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}-${{ steps.det-params.outputs.dep_hash }} restore-keys: | + ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}- ${{ runner.os }}-depends-${{ matrix.depends_name }}- - name: Build depends @@ -107,32 +113,7 @@ jobs: runs-on: ubuntu-22.04 strategy: fail-fast: false - matrix: - include: - - build_target: arm-linux - depends_name: arm-linux-gnueabihf - host: arm-linux-gnueabihf - - build_target: linux64 - depends_name: x86_64-pc-linux-gnu_debug - host: x86_64-pc-linux-gnu - - build_target: linux64_cxx20 - depends_name: x86_64-pc-linux-gnu_debug - host: x86_64-pc-linux-gnu - - build_target: linux64_fuzz - depends_name: x86_64-pc-linux-gnu_debug - host: x86_64-pc-linux-gnu - - build_target: linux64_nowallet - depends_name: x86_64-pc-linux-gnu_debug - host: x86_64-pc-linux-gnu - - build_target: linux64_sqlite - depends_name: x86_64-pc-linux-gnu_debug - host: x86_64-pc-linux-gnu - - build_target: linux64_tsan - depends_name: x86_64-pc-linux-gnu_debug - host: x86_64-pc-linux-gnu - - build_target: linux64_ubsan - depends_name: x86_64-pc-linux-gnu_debug - host: x86_64-pc-linux-gnu + matrix: ${{ fromJson(needs.build-image.outputs.src-matrix) }} container: image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} options: --user root @@ -142,14 +123,27 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} + - name: Determine params + id: det-params + run: | + dep_name="${{ matrix.depends_name }}" + dep_opts="$(jq -r ".dep.include[] | select(.depends_name == \"${{ matrix.depends_name }}\") | .dep_opts" -c .github/workflows/matrix.json)" + dep_hash="$(echo -n ${dep_opts} | sha256sum | head -c 64)" + echo "\"${dep_name}\" has DEP_OPTS \"${dep_opts}\" with hash \"${dep_hash}\"" + echo "dep_hash=${dep_hash}" >> $GITHUB_OUTPUT + dep_host="$(jq -r ".dep.include[] | select(.depends_name == \"${{ matrix.depends_name }}\") | .host" -c .github/workflows/matrix.json)" + echo "\"${dep_name}\" has HOST \"${dep_host}\"" + echo "dep_host=${dep_host}" >> $GITHUB_OUTPUT + - name: Restore depends cache uses: actions/cache/restore@v4 with: path: | depends/built - depends/${{ matrix.host }} - key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }} + depends/${{ steps.det-params.outputs.dep_host }} + key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}-${{ steps.det-params.outputs.dep_hash }} restore-keys: | + ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}- ${{ runner.os }}-depends-${{ matrix.depends_name }}- - name: Determine PR Base SHA diff --git a/.github/workflows/matrix.json b/.github/workflows/matrix.json new file mode 100644 index 0000000000000..dcfcd735aaca0 --- /dev/null +++ b/.github/workflows/matrix.json @@ -0,0 +1,52 @@ +{ + "dep": { + "include": [ + { + "depends_name": "arm-linux-gnueabihf", + "host": "arm-linux-gnueabihf", + "dep_opts": "" + }, + { + "depends_name": "x86_64-pc-linux-gnu_debug", + "host": "x86_64-pc-linux-gnu", + "dep_opts": "DEBUG=1" + } + ] + }, + "src": { + "include": [ + { + "build_target": "arm-linux", + "depends_name": "arm-linux-gnueabihf" + }, + { + "build_target": "linux64", + "depends_name": "x86_64-pc-linux-gnu_debug" + }, + { + "build_target": "linux64_cxx20", + "depends_name": "x86_64-pc-linux-gnu_debug" + }, + { + "build_target": "linux64_fuzz", + "depends_name": "x86_64-pc-linux-gnu_debug" + }, + { + "build_target": "linux64_nowallet", + "depends_name": "x86_64-pc-linux-gnu_debug" + }, + { + "build_target": "linux64_sqlite", + "depends_name": "x86_64-pc-linux-gnu_debug" + }, + { + "build_target": "linux64_tsan", + "depends_name": "x86_64-pc-linux-gnu_debug" + }, + { + "build_target": "linux64_ubsan", + "depends_name": "x86_64-pc-linux-gnu_debug" + } + ] + } +} From 239b7c0b43753af253ad648791680252db8fddac Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:41:05 +0000 Subject: [PATCH 08/15] ci: add GitLab nowallet depends, multiprocess deps+build to GitHub runs --- .github/workflows/matrix.json | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/matrix.json b/.github/workflows/matrix.json index dcfcd735aaca0..9440abcc91993 100644 --- a/.github/workflows/matrix.json +++ b/.github/workflows/matrix.json @@ -10,6 +10,16 @@ "depends_name": "x86_64-pc-linux-gnu_debug", "host": "x86_64-pc-linux-gnu", "dep_opts": "DEBUG=1" + }, + { + "depends_name": "x86_64-pc-linux-gnu_multiprocess", + "host": "x86_64-pc-linux-gnu", + "dep_opts": "MULTIPROCESS=1" + }, + { + "depends_name": "x86_64-pc-linux-gnu_nowallet", + "host": "x86_64-pc-linux-gnu", + "dep_opts": "NO_WALLET=1" } ] }, @@ -31,9 +41,13 @@ "build_target": "linux64_fuzz", "depends_name": "x86_64-pc-linux-gnu_debug" }, + { + "build_target": "linux64_multiprocess", + "depends_name": "x86_64-pc-linux-gnu_multiprocess" + }, { "build_target": "linux64_nowallet", - "depends_name": "x86_64-pc-linux-gnu_debug" + "depends_name": "x86_64-pc-linux-gnu_nowallet" }, { "build_target": "linux64_sqlite", From 86412870d301ceda3e30f2fc177e94bc973dc777 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 19 Nov 2024 08:20:56 +0000 Subject: [PATCH 09/15] fix: add `linux64_multiprocess` BUILD_TARGET to `matrix.sh`, mend C(XX) --- .github/workflows/matrix.json | 2 +- .gitlab-ci.yml | 2 +- ci/dash/matrix.sh | 2 ++ ci/test/00_setup_env_native_multiprocess.sh | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/matrix.json b/.github/workflows/matrix.json index 9440abcc91993..2ed63717eb426 100644 --- a/.github/workflows/matrix.json +++ b/.github/workflows/matrix.json @@ -14,7 +14,7 @@ { "depends_name": "x86_64-pc-linux-gnu_multiprocess", "host": "x86_64-pc-linux-gnu", - "dep_opts": "MULTIPROCESS=1" + "dep_opts": "DEBUG=1 MULTIPROCESS=1" }, { "depends_name": "x86_64-pc-linux-gnu_nowallet", diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 15200ef9441ec..264e5efcf397e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -209,7 +209,7 @@ x86_64-pc-linux-gnu_multiprocess: - .skip-in-fast-mode-template variables: HOST: x86_64-pc-linux-gnu - DEP_OPTS: "MULTIPROCESS=1" + DEP_OPTS: "DEBUG=1 MULTIPROCESS=1" x86_64-apple-darwin: extends: diff --git a/ci/dash/matrix.sh b/ci/dash/matrix.sh index 6eeb2a7565317..9822a727890b5 100755 --- a/ci/dash/matrix.sh +++ b/ci/dash/matrix.sh @@ -26,6 +26,8 @@ elif [ "$BUILD_TARGET" = "linux64_cxx20" ]; then source ./ci/test/00_setup_env_native_cxx20.sh elif [ "$BUILD_TARGET" = "linux64_fuzz" ]; then source ./ci/test/00_setup_env_native_fuzz.sh +elif [ "$BUILD_TARGET" = "linux64_multiprocess" ]; then + source ./ci/test/00_setup_env_native_multiprocess.sh elif [ "$BUILD_TARGET" = "linux64_nowallet" ]; then source ./ci/test/00_setup_env_native_nowallet.sh elif [ "$BUILD_TARGET" = "linux64_sqlite" ]; then diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index d6727d05b0611..06f5eeb481573 100755 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -10,5 +10,5 @@ export CONTAINER_NAME=ci_native_multiprocess export PACKAGES="cmake python3 llvm clang" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" -export BITCOIN_CONFIG="--with-boost-process --enable-debug CC=clang CXX=clang++" # Use clang to avoid OOM +export BITCOIN_CONFIG="--with-boost-process --enable-debug CC=clang-16 CXX=clang++-16" # Use clang to avoid OOM export BITCOIND=dash-node # Used in functional tests From 732322bb0b44b24b546773d6bc51daf6e01842e7 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Tue, 19 Nov 2024 14:12:00 +0000 Subject: [PATCH 10/15] ci: build multiprocess and tsan depends with Clang, drop process in fuzz --- .github/workflows/matrix.json | 4 ++-- .gitlab-ci.yml | 4 ++-- ci/test/00_setup_env_native_fuzz.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/matrix.json b/.github/workflows/matrix.json index 2ed63717eb426..4560bab2e73a6 100644 --- a/.github/workflows/matrix.json +++ b/.github/workflows/matrix.json @@ -14,7 +14,7 @@ { "depends_name": "x86_64-pc-linux-gnu_multiprocess", "host": "x86_64-pc-linux-gnu", - "dep_opts": "DEBUG=1 MULTIPROCESS=1" + "dep_opts": "DEBUG=1 MULTIPROCESS=1 CC=clang-16 CXX=clang++-16" }, { "depends_name": "x86_64-pc-linux-gnu_nowallet", @@ -55,7 +55,7 @@ }, { "build_target": "linux64_tsan", - "depends_name": "x86_64-pc-linux-gnu_debug" + "depends_name": "x86_64-pc-linux-gnu_multiprocess" }, { "build_target": "linux64_ubsan", diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 264e5efcf397e..743a2164b559a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -209,7 +209,7 @@ x86_64-pc-linux-gnu_multiprocess: - .skip-in-fast-mode-template variables: HOST: x86_64-pc-linux-gnu - DEP_OPTS: "DEBUG=1 MULTIPROCESS=1" + DEP_OPTS: "DEBUG=1 MULTIPROCESS=1 CC=clang-16 CXX=clang++-16" x86_64-apple-darwin: extends: @@ -284,7 +284,7 @@ linux64_tsan-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu_debug + - x86_64-pc-linux-gnu_multiprocess variables: BUILD_TARGET: linux64_tsan diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 2ba78325ff16c..8c14c618dfa2d 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -16,4 +16,4 @@ export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer --enable-suppress-external-warnings CC=clang-16 CXX=clang++-16 --with-boost-process" +export BITCOIN_CONFIG="--enable-zmq --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer --enable-suppress-external-warnings CC=clang-16 CXX=clang++-16" From ca2e95a164b9a27ea4a499b388af86f885c00100 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 20 Nov 2024 04:23:38 +0000 Subject: [PATCH 11/15] ci: drop distro LLVM packages, move Clang install up, set defaults --- contrib/containers/ci/Dockerfile | 48 +++++++++++++++++++------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index f80a1b5174b5c..c4125752e2620 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -7,7 +7,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" # (zlib1g-dev is needed for the Qt host binary builds, but should not be used by target binaries) ENV APT_ARGS="-y --no-install-recommends --no-upgrade" - +SHELL ["/bin/bash", "-c"] # Install packages for i386; disabled on aarch64 and arm64 hosts RUN (dpkg --print-architecture | grep -Eq 'aarch64|arm64' || dpkg --add-architecture i386) RUN (dpkg --print-architecture | grep -Eq 'aarch64|arm64' || (apt-get update && apt-get install $APT_ARGS \ @@ -23,13 +23,11 @@ RUN apt-get update && apt-get install $APT_ARGS \ bsdmainutils \ curl \ ccache \ - clang \ cmake \ g++ \ gettext \ git \ - libc++-dev \ - libc++abi-dev \ + gnupg \ libtool \ libxcb-icccm4 \ libxcb-image0 \ @@ -42,12 +40,37 @@ RUN apt-get update && apt-get install $APT_ARGS \ libxcb-xinerama0 \ libxcb-xkb1 \ libxkbcommon-x11-0 \ - wget \ + lsb-release \ + software-properties-common \ unzip \ + wget \ m4 \ pkg-config \ zlib1g-dev +# Install Clang+LLVM and set it as default +# We don't need all packages but the default set doesn't include some +# packages we want so we will need to install some of them manually. +ARG LLVM_VERSION=16 +RUN set -ex; \ + cd /tmp && \ + wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + /tmp/llvm.sh ${LLVM_VERSION} && \ + rm -rf /tmp/llvm.sh; \ + apt-get update && apt-get install $APT_ARGS \ + clang-format-${LLVM_VERSION} \ + clang-tidy-${LLVM_VERSION} \ + libc++-${LLVM_VERSION}-dev \ + libc++abi-${LLVM_VERSION}-dev \ + libclang-rt-${LLVM_VERSION}-dev; \ + rm -rf /var/lib/apt/lists/*; \ + lldbUpdAltArgs="update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${LLVM_VERSION} 100"; \ + for binName in clang clang++ clang-format clang-tidy clangd ld.lld lldb lldb-server; do \ + lldbUpdAltArgs="${lldbUpdAltArgs} --slave /usr/bin/${binName} ${binName} /usr/bin/${binName}-${LLVM_VERSION}"; \ + done; \ + bash -c "${lldbUpdAltArgs}"; + # Python setup # PYTHON_VERSION should match the value in .python-version ARG PYTHON_VERSION=3.9.18 @@ -61,7 +84,6 @@ RUN apt-get update && apt-get install $APT_ARGS \ libreadline-dev \ libsqlite3-dev \ libssl-dev \ - llvm \ make \ tk-dev \ xz-utils @@ -136,20 +158,6 @@ RUN \ update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; \ exit 0 -ARG LLVM_VERSION=16 -# Setup Clang+LLVM support -RUN apt-get update && apt-get install $APT_ARGS \ - lsb-release \ - software-properties-common \ - gnupg \ - && rm -rf /var/lib/apt/lists/* - -RUN cd /tmp && \ - wget https://apt.llvm.org/llvm.sh && \ - chmod +x llvm.sh && \ - /tmp/llvm.sh ${LLVM_VERSION} && \ - rm -rf /tmp/llvm.sh - RUN \ mkdir -p /src/dash && \ mkdir -p /cache/ccache && \ From 77795a56cc8f426aac5f17ff0e28db7595f32cbc Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:40:42 +0000 Subject: [PATCH 12/15] ci: bump container to Clang 18, update CI --- .github/workflows/matrix.json | 2 +- .gitlab-ci.yml | 2 +- ci/test/00_setup_env_native_fuzz.sh | 2 +- ci/test/00_setup_env_native_fuzz_with_valgrind.sh | 2 +- ci/test/00_setup_env_native_multiprocess.sh | 2 +- ci/test/00_setup_env_native_tsan.sh | 6 +++--- ci/test/00_setup_env_native_ubsan.sh | 4 ++-- ci/test/00_setup_env_native_valgrind.sh | 2 +- contrib/containers/ci/Dockerfile | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/matrix.json b/.github/workflows/matrix.json index 4560bab2e73a6..dd3cd1b4d988f 100644 --- a/.github/workflows/matrix.json +++ b/.github/workflows/matrix.json @@ -14,7 +14,7 @@ { "depends_name": "x86_64-pc-linux-gnu_multiprocess", "host": "x86_64-pc-linux-gnu", - "dep_opts": "DEBUG=1 MULTIPROCESS=1 CC=clang-16 CXX=clang++-16" + "dep_opts": "DEBUG=1 MULTIPROCESS=1 CC=clang-18 CXX=clang++-18" }, { "depends_name": "x86_64-pc-linux-gnu_nowallet", diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 743a2164b559a..477d1f01a84dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -209,7 +209,7 @@ x86_64-pc-linux-gnu_multiprocess: - .skip-in-fast-mode-template variables: HOST: x86_64-pc-linux-gnu - DEP_OPTS: "DEBUG=1 MULTIPROCESS=1 CC=clang-16 CXX=clang++-16" + DEP_OPTS: "DEBUG=1 MULTIPROCESS=1 CC=clang-18 CXX=clang++-18" x86_64-apple-darwin: extends: diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 8c14c618dfa2d..ad69665b6aec7 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -16,4 +16,4 @@ export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer --enable-suppress-external-warnings CC=clang-16 CXX=clang++-16" +export BITCOIN_CONFIG="--enable-zmq --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer --enable-suppress-external-warnings CC=clang-18 CXX=clang++-18" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index b5c6b60b4a73a..9860123546744 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -14,4 +14,4 @@ export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer --enable-suppress-external-warnings CC=clang-16 CXX=clang++-16" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer --enable-suppress-external-warnings CC=clang-18 CXX=clang++-18" diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index 06f5eeb481573..b63f7aeb6f79a 100755 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -10,5 +10,5 @@ export CONTAINER_NAME=ci_native_multiprocess export PACKAGES="cmake python3 llvm clang" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" -export BITCOIN_CONFIG="--with-boost-process --enable-debug CC=clang-16 CXX=clang++-16" # Use clang to avoid OOM +export BITCOIN_CONFIG="--with-boost-process --enable-debug CC=clang-18 CXX=clang++-18" # Use clang to avoid OOM export BITCOIND=dash-node # Used in functional tests diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 633304580f40e..ffa3d8365c858 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -7,11 +7,11 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libc++abi-16-dev libc++-16-dev python3-zmq" -export DEP_OPTS="CC=clang-16 CXX='clang++-16 -stdlib=libc++'" +export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libc++abi-18-dev libc++-18-dev python3-zmq" +export DEP_OPTS="CC=clang-18 CXX='clang++-18 -stdlib=libc++'" export TEST_RUNNER_EXTRA="--extended --exclude feature_pruning,feature_dbcrash,wallet_multiwallet.py" # Temporarily suppress ASan heap-use-after-free (see issue #14163) export TEST_RUNNER_EXTRA="${TEST_RUNNER_EXTRA} --timeout-factor=4" # Increase timeout because sanitizers slow down export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-gui=no --with-sanitizers=thread CC=clang-16 CXX=clang++-16 CXXFLAGS='-g' --with-boost-process" +export BITCOIN_CONFIG="--enable-zmq --with-gui=no --with-sanitizers=thread CC=clang-18 CXX=clang++-18 CXXFLAGS='-g' --with-boost-process" export CPPFLAGS="-DDEBUG_LOCKORDER -DARENA_DEBUG" export PYZMQ=true diff --git a/ci/test/00_setup_env_native_ubsan.sh b/ci/test/00_setup_env_native_ubsan.sh index 3d44490a4f102..9fb440dfd41e2 100755 --- a/ci/test/00_setup_env_native_ubsan.sh +++ b/ci/test/00_setup_env_native_ubsan.sh @@ -8,8 +8,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_ubsan -export PACKAGES="clang-16 llvm-16 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export PACKAGES="clang-18 llvm-18 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export DEP_OPTS="NO_UPNP=1 DEBUG=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --enable-reduce-exports --enable-crash-hooks --with-sanitizers=undefined CC=clang-16 CXX=clang++-16" +export BITCOIN_CONFIG="--enable-zmq --enable-reduce-exports --enable-crash-hooks --with-sanitizers=undefined CC=clang-18 CXX=clang++-18" export PYZMQ=true diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 2703963a05549..b0793313fb1a5 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -11,4 +11,4 @@ export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--exclude rpc_bind --timeout-factor=4" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no --enable-suppress-external-warnings CC=clang-16 CXX=clang++-16" # TODO enable GUI +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no --enable-suppress-external-warnings CC=clang-18 CXX=clang++-18" # TODO enable GUI diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index c4125752e2620..fb3f5923352e2 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install $APT_ARGS \ # Install Clang+LLVM and set it as default # We don't need all packages but the default set doesn't include some # packages we want so we will need to install some of them manually. -ARG LLVM_VERSION=16 +ARG LLVM_VERSION=18 RUN set -ex; \ cd /tmp && \ wget https://apt.llvm.org/llvm.sh && \ From 3cc8c70fcc6d14eeddb3600f815a8e8643d47ee7 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sat, 9 Nov 2024 10:05:33 +0000 Subject: [PATCH 13/15] ci: remove redundant `version` attribute, avoid `lldb` personality error --- contrib/containers/develop/docker-compose.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/containers/develop/docker-compose.yml b/contrib/containers/develop/docker-compose.yml index 95241c0a563a2..9238c97183e79 100644 --- a/contrib/containers/develop/docker-compose.yml +++ b/contrib/containers/develop/docker-compose.yml @@ -1,17 +1,18 @@ -version: "3.9" services: container: entrypoint: /bin/bash build: context: '..' dockerfile: './develop/Dockerfile' - tty: true # Equivalent to -t - stdin_open: true # Equivalent to -i ports: - "9998:9998" # Mainnet Ports - "9999:9999" - "19998:19998" # Testnet Ports - "19999:19999" + security_opt: + - seccomp:unconfined + stdin_open: true # Equivalent to -i + tty: true # Equivalent to -t # A note about volumes: # From 139c785e5398239de44092532327805c25b07a1b Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Sat, 23 Nov 2024 14:13:49 +0300 Subject: [PATCH 14/15] ci: Drop matrix.json, align CI-s --- .github/workflows/build.yml | 66 +++++++++++++++------ .github/workflows/matrix.json | 66 --------------------- .gitlab-ci.yml | 20 ++++--- ci/test/00_setup_env_native_multiprocess.sh | 2 +- 4 files changed, 59 insertions(+), 95 deletions(-) delete mode 100644 .github/workflows/matrix.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7e39b112306f1..9a0a28d11f82c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,8 +19,6 @@ jobs: outputs: image-tag: ${{ steps.prepare.outputs.image-tag }} repo-name: ${{ steps.prepare.outputs.repo-name }} - dep-matrix: ${{ steps.prepare.outputs.dep-matrix }} - src-matrix: ${{ steps.prepare.outputs.src-matrix }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -34,8 +32,6 @@ jobs: REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') echo "image-tag=${BRANCH_NAME}" >> $GITHUB_OUTPUT echo "repo-name=${REPO_NAME}" >> $GITHUB_OUTPUT - echo "dep-matrix=$(jq -r '.dep' -c .github/workflows/matrix.json)" >> $GITHUB_OUTPUT - echo "src-matrix=$(jq -r '.src' -c .github/workflows/matrix.json)" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -65,7 +61,16 @@ jobs: runs-on: ubuntu-22.04 strategy: fail-fast: false - matrix: ${{ fromJson(needs.build-image.outputs.dep-matrix) }} + matrix: + include: + - depends_name: arm-linux-gnueabihf + build_target: arm-linux + - depends_name: x86_64-pc-linux-gnu_debug + build_target: linux64 + - depends_name: x86_64-pc-linux-gnu_multiprocess + build_target: linux64_multiprocess + - depends_name: x86_64-pc-linux-gnu_nowallet + build_target: linux64_nowallet container: image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} options: --user root @@ -87,25 +92,29 @@ jobs: - name: Determine params id: det-params run: | - dep_name="${{ matrix.depends_name }}" - dep_opts="${{ matrix.dep_opts }}" - dep_hash="$(echo -n ${dep_opts} | sha256sum | head -c 64)" - echo "\"${dep_name}\" has DEP_OPTS \"${dep_opts}\" with hash \"${dep_hash}\"" + BUILD_TARGET="${{ matrix.build_target }}" + source ./ci/dash/matrix.sh + echo "dep_opts=${DEP_OPTS}" >> $GITHUB_OUTPUT + dep_hash="$(echo -n ${DEP_OPTS} | sha256sum | head -c 64)" + echo "\"${{ matrix.depends_name }}\" has DEP_OPTS \"${DEP_OPTS}\" with hash \"${dep_hash}\"" echo "dep_hash=${dep_hash}" >> $GITHUB_OUTPUT + echo "\"${{ matrix.depends_name }}\" has HOST \"${HOST}\"" + echo "dep_host=${HOST}" >> $GITHUB_OUTPUT + shell: bash - name: Cache depends uses: actions/cache@v4 with: path: | depends/built - depends/${{ matrix.host }} + depends/${{ steps.det-params.outputs.dep_host }} key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}-${{ steps.det-params.outputs.dep_hash }} restore-keys: | ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}- ${{ runner.os }}-depends-${{ matrix.depends_name }}- - name: Build depends - run: env HOST=${{ matrix.host }} ${{ matrix.dep_opts }} make -j$(nproc) -C depends + run: env HOST=${{ steps.det-params.outputs.dep_host }} ${{ steps.det-params.outputs.dep_opts }} make -j$(nproc) -C depends build: name: ${{ matrix.build_target }} @@ -113,7 +122,26 @@ jobs: runs-on: ubuntu-22.04 strategy: fail-fast: false - matrix: ${{ fromJson(needs.build-image.outputs.src-matrix) }} + matrix: + include: + - build_target: arm-linux + depends_name: arm-linux-gnueabihf + - build_target: linux64 + depends_name: x86_64-pc-linux-gnu_debug + - build_target: linux64_cxx20 + depends_name: x86_64-pc-linux-gnu_debug + - build_target: linux64_fuzz + depends_name: x86_64-pc-linux-gnu_debug + - build_target: linux64_multiprocess + depends_name: x86_64-pc-linux-gnu_multiprocess + - build_target: linux64_nowallet + depends_name: x86_64-pc-linux-gnu_nowallet + - build_target: linux64_sqlite + depends_name: x86_64-pc-linux-gnu_debug + - build_target: linux64_tsan + depends_name: x86_64-pc-linux-gnu_debug + - build_target: linux64_ubsan + depends_name: x86_64-pc-linux-gnu_debug container: image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} options: --user root @@ -126,14 +154,14 @@ jobs: - name: Determine params id: det-params run: | - dep_name="${{ matrix.depends_name }}" - dep_opts="$(jq -r ".dep.include[] | select(.depends_name == \"${{ matrix.depends_name }}\") | .dep_opts" -c .github/workflows/matrix.json)" - dep_hash="$(echo -n ${dep_opts} | sha256sum | head -c 64)" - echo "\"${dep_name}\" has DEP_OPTS \"${dep_opts}\" with hash \"${dep_hash}\"" + BUILD_TARGET="${{ matrix.build_target }}" + source ./ci/dash/matrix.sh + dep_hash="$(echo -n ${DEP_OPTS} | sha256sum | head -c 64)" + echo "\"${{ matrix.build_target }}\" has DEP_OPTS \"${DEP_OPTS}\" with hash \"${dep_hash}\"" echo "dep_hash=${dep_hash}" >> $GITHUB_OUTPUT - dep_host="$(jq -r ".dep.include[] | select(.depends_name == \"${{ matrix.depends_name }}\") | .host" -c .github/workflows/matrix.json)" - echo "\"${dep_name}\" has HOST \"${dep_host}\"" - echo "dep_host=${dep_host}" >> $GITHUB_OUTPUT + echo "\"${{ matrix.build_target }}\" has HOST \"${HOST}\"" + echo "dep_host=${HOST}" >> $GITHUB_OUTPUT + shell: bash - name: Restore depends cache uses: actions/cache/restore@v4 diff --git a/.github/workflows/matrix.json b/.github/workflows/matrix.json deleted file mode 100644 index dd3cd1b4d988f..0000000000000 --- a/.github/workflows/matrix.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "dep": { - "include": [ - { - "depends_name": "arm-linux-gnueabihf", - "host": "arm-linux-gnueabihf", - "dep_opts": "" - }, - { - "depends_name": "x86_64-pc-linux-gnu_debug", - "host": "x86_64-pc-linux-gnu", - "dep_opts": "DEBUG=1" - }, - { - "depends_name": "x86_64-pc-linux-gnu_multiprocess", - "host": "x86_64-pc-linux-gnu", - "dep_opts": "DEBUG=1 MULTIPROCESS=1 CC=clang-18 CXX=clang++-18" - }, - { - "depends_name": "x86_64-pc-linux-gnu_nowallet", - "host": "x86_64-pc-linux-gnu", - "dep_opts": "NO_WALLET=1" - } - ] - }, - "src": { - "include": [ - { - "build_target": "arm-linux", - "depends_name": "arm-linux-gnueabihf" - }, - { - "build_target": "linux64", - "depends_name": "x86_64-pc-linux-gnu_debug" - }, - { - "build_target": "linux64_cxx20", - "depends_name": "x86_64-pc-linux-gnu_debug" - }, - { - "build_target": "linux64_fuzz", - "depends_name": "x86_64-pc-linux-gnu_debug" - }, - { - "build_target": "linux64_multiprocess", - "depends_name": "x86_64-pc-linux-gnu_multiprocess" - }, - { - "build_target": "linux64_nowallet", - "depends_name": "x86_64-pc-linux-gnu_nowallet" - }, - { - "build_target": "linux64_sqlite", - "depends_name": "x86_64-pc-linux-gnu_debug" - }, - { - "build_target": "linux64_tsan", - "depends_name": "x86_64-pc-linux-gnu_multiprocess" - }, - { - "build_target": "linux64_ubsan", - "depends_name": "x86_64-pc-linux-gnu_debug" - } - ] - } -} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 477d1f01a84dc..4d36281248406 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -45,7 +45,10 @@ builder-image: XCODE_VERSION: "15.0" XCODE_BUILD_ID: 15A240d before_script: + - echo BUILD_TARGET=$BUILD_TARGET + - source ./ci/dash/matrix.sh - echo HOST=$HOST + - echo DEP_OPTS=$DEP_OPTS - | if [ "$HOST" = "x86_64-apple-darwin" ]; then mkdir -p depends/SDKs @@ -87,6 +90,8 @@ builder-image: - export CACHE_DIR=$CI_PROJECT_DIR/cache - echo BUILD_TARGET=$BUILD_TARGET - source ./ci/dash/matrix.sh + - echo HOST=$HOST + - echo DEP_OPTS=$DEP_OPTS # Setup some environment variables - | @@ -180,43 +185,40 @@ builder-image: arm-linux-gnueabihf: extends: .build-depends-template variables: - HOST: arm-linux-gnueabihf + BUILD_TARGET: arm-linux x86_64-w64-mingw32: extends: - .build-depends-template - .skip-in-fast-mode-template variables: - HOST: x86_64-w64-mingw32 + BUILD_TARGET: win64 x86_64-pc-linux-gnu_debug: extends: .build-depends-template variables: - HOST: x86_64-pc-linux-gnu - DEP_OPTS: "DEBUG=1" + BUILD_TARGET: linux64 x86_64-pc-linux-gnu_nowallet: extends: - .build-depends-template - .skip-in-fast-mode-template variables: - HOST: x86_64-pc-linux-gnu - DEP_OPTS: "NO_WALLET=1" + BUILD_TARGET: linux64_nowallet x86_64-pc-linux-gnu_multiprocess: extends: - .build-depends-template - .skip-in-fast-mode-template variables: - HOST: x86_64-pc-linux-gnu - DEP_OPTS: "DEBUG=1 MULTIPROCESS=1 CC=clang-18 CXX=clang++-18" + BUILD_TARGET: linux64_multiprocess x86_64-apple-darwin: extends: - .build-depends-template - .skip-in-fast-mode-template variables: - HOST: x86_64-apple-darwin + BUILD_TARGET: mac ### diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index b63f7aeb6f79a..5b0f88f076033 100755 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_multiprocess export PACKAGES="cmake python3 llvm clang" -export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" +export DEP_OPTS="DEBUG=1 MULTIPROCESS=1 CC=clang-18 CXX=clang++-18" export GOAL="install" export BITCOIN_CONFIG="--with-boost-process --enable-debug CC=clang-18 CXX=clang++-18" # Use clang to avoid OOM export BITCOIND=dash-node # Used in functional tests From 94e37efe24f8536ab7969703814eb98f68de4ebb Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Sun, 24 Nov 2024 20:16:58 +0300 Subject: [PATCH 15/15] ci: build and use sqlite and tsan depends NOTE: no '-stdlib=libc++' in CXX for DEP_OPTS and BITCOIN_CONFIG for now --- .github/workflows/build.yml | 8 ++++++-- .gitlab-ci.yml | 18 ++++++++++++++++-- ci/test/00_setup_env_native_tsan.sh | 3 ++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9a0a28d11f82c..666547796f16f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,8 +69,12 @@ jobs: build_target: linux64 - depends_name: x86_64-pc-linux-gnu_multiprocess build_target: linux64_multiprocess + - depends_name: x86_64-pc-linux-gnu_sqlite + build_target: linux64_sqlite - depends_name: x86_64-pc-linux-gnu_nowallet build_target: linux64_nowallet + - depends_name: x86_64-pc-linux-gnu_tsan + build_target: linux64_tsan container: image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} options: --user root @@ -137,9 +141,9 @@ jobs: - build_target: linux64_nowallet depends_name: x86_64-pc-linux-gnu_nowallet - build_target: linux64_sqlite - depends_name: x86_64-pc-linux-gnu_debug + depends_name: x86_64-pc-linux-gnu_sqlite - build_target: linux64_tsan - depends_name: x86_64-pc-linux-gnu_debug + depends_name: x86_64-pc-linux-gnu_tsan - build_target: linux64_ubsan depends_name: x86_64-pc-linux-gnu_debug container: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d36281248406..1522db3343958 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -206,6 +206,13 @@ x86_64-pc-linux-gnu_nowallet: variables: BUILD_TARGET: linux64_nowallet +x86_64-pc-linux-gnu_sqlite: + extends: + - .build-depends-template + - .skip-in-fast-mode-template + variables: + BUILD_TARGET: linux64_sqlite + x86_64-pc-linux-gnu_multiprocess: extends: - .build-depends-template @@ -213,6 +220,13 @@ x86_64-pc-linux-gnu_multiprocess: variables: BUILD_TARGET: linux64_multiprocess +x86_64-pc-linux-gnu_tsan: + extends: + - .build-depends-template + - .skip-in-fast-mode-template + variables: + BUILD_TARGET: linux64_tsan + x86_64-apple-darwin: extends: - .build-depends-template @@ -259,7 +273,7 @@ linux64_sqlite-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu_debug + - x86_64-pc-linux-gnu_sqlite variables: BUILD_TARGET: linux64_sqlite @@ -286,7 +300,7 @@ linux64_tsan-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu_multiprocess + - x86_64-pc-linux-gnu_tsan variables: BUILD_TARGET: linux64_tsan diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index ffa3d8365c858..cf1ace46cedea 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -8,7 +8,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libc++abi-18-dev libc++-18-dev python3-zmq" -export DEP_OPTS="CC=clang-18 CXX='clang++-18 -stdlib=libc++'" +# NOTE: no '-stdlib=libc++' in CXX for DEP_OPTS and BITCOIN_CONFIG for now +export DEP_OPTS="CC=clang-18 CXX=clang++-18" export TEST_RUNNER_EXTRA="--extended --exclude feature_pruning,feature_dbcrash,wallet_multiwallet.py" # Temporarily suppress ASan heap-use-after-free (see issue #14163) export TEST_RUNNER_EXTRA="${TEST_RUNNER_EXTRA} --timeout-factor=4" # Increase timeout because sanitizers slow down export GOAL="install"