diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ae6d83ddd3..e6c5ad3ac2e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,7 @@ on: pull_request: branches-ignore: - master + types: [opened, synchronize, reopened, labeled, unlabeled] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -26,19 +27,33 @@ jobs: cd ext-functional-test-demo cargo test --verbose --color always cargo test --verbose --color always --features test-broken + # Determines CI configuration based on trigger type. + # - Push to main or full-test label: full matrix + all steps (incl. fuzz, coverage, benchmark) + # - Other pushes/pull_requests: self-hosted/1.75.0 + core steps only + ci-config: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.config.outputs.matrix }} + run_all: ${{ steps.config.outputs.run_all }} + steps: + - name: Determine matrix and scope + id: config + run: | + FULL_MATRIX='{"include":[{"platform":"self-hosted","toolchain":"stable"},{"platform":"self-hosted","toolchain":"beta"},{"platform":"self-hosted","toolchain":"1.75.0"},{"platform":"windows-latest","toolchain":"stable"},{"platform":"macos-latest","toolchain":"stable"},{"platform":"macos-latest","toolchain":"1.75.0"}]}' + CORE_MATRIX='{"include":[{"platform":"self-hosted","toolchain":"1.75.0"}]}' + if [[ "${{ github.ref }}" == "refs/heads/main" || "${{ contains(github.event.pull_request.labels.*.name, 'full-test') }}" == "true" ]]; then + echo "matrix=$FULL_MATRIX" >> "$GITHUB_OUTPUT" + echo "run_all=true" >> "$GITHUB_OUTPUT" + else + echo "matrix=$CORE_MATRIX" >> "$GITHUB_OUTPUT" + echo "run_all=false" >> "$GITHUB_OUTPUT" + fi + build: + needs: ci-config strategy: fail-fast: false - matrix: - platform: [ self-hosted, windows-latest, macos-latest ] - toolchain: [ stable, beta, 1.75.0 ] # 1.75.0 is the MSRV for all crates - exclude: - - platform: windows-latest - toolchain: 1.75.0 - - platform: windows-latest - toolchain: beta - - platform: macos-latest - toolchain: beta + matrix: ${{ fromJson(needs.ci-config.outputs.matrix) }} runs-on: ${{ matrix.platform }} steps: - name: Checkout source code @@ -88,12 +103,89 @@ jobs: run: | echo "BITCOIND_EXE=$( pwd )/bin/bitcoind-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV" echo "ELECTRS_EXE=$( pwd )/bin/electrs-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV" - - name: Run CI script - shell: bash # Default on Winblows is powershell - run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh + # --- Core steps (always run) --- + - name: Check workspace (except lightning-transaction-sync) + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh check-workspace + - name: Test workspace (except lightning-transaction-sync) + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-workspace + - name: Test upgrade from prior LDK versions + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-ldk-upgrade + - name: Check and build docs for workspace members + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh check-workspace-members + # --- Label-gated: block-sync --- + - name: Test lightning with dnssec feature + if: needs.ci-config.outputs.run_all == 'true' || contains(github.event.pull_request.labels.*.name, 'block-sync') + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-lightning-dnssec + - name: Test Block Sync Clients with features + if: needs.ci-config.outputs.run_all == 'true' || contains(github.event.pull_request.labels.*.name, 'block-sync') + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-lightning-block-sync + - name: Check Transaction Sync Clients + if: needs.ci-config.outputs.run_all == 'true' || contains(github.event.pull_request.labels.*.name, 'block-sync') + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh check-lightning-transaction-sync + - name: Test Transaction Sync Clients + if: needs.ci-config.outputs.run_all == 'true' || contains(github.event.pull_request.labels.*.name, 'block-sync') + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-lightning-transaction-sync + # --- Label-gated: full-test --- + - name: Test lightning-persister with tokio + if: needs.ci-config.outputs.run_all == 'true' + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-lightning-persister + - name: Test Custom Message Macros + if: needs.ci-config.outputs.run_all == 'true' + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-lightning-custom-message + - name: Test backtrace-debug builds + if: needs.ci-config.outputs.run_all == 'true' + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-lightning-backtrace + - name: Test other crate-specific builds + if: needs.ci-config.outputs.run_all == 'true' + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-crate-specific + # --- Label-gated: no-std --- + - name: Test no_std builds + if: needs.ci-config.outputs.run_all == 'true' || contains(github.event.pull_request.labels.*.name, 'no-std') + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-no-std + - name: Check no_std downstream crate + if: needs.ci-config.outputs.run_all == 'true' || contains(github.event.pull_request.labels.*.name, 'no-std') + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh check-no-std + - name: Check MSRV with release pins only + if: needs.ci-config.outputs.run_all == 'true' || contains(github.event.pull_request.labels.*.name, 'no-std') + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh check-msrv-no-dev-deps + - name: Build no_std for ARM Embedded + if: needs.ci-config.outputs.run_all == 'true' || contains(github.event.pull_request.labels.*.name, 'no-std') + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh build-no-std-arm + # --- Label-gated: c-bindings --- + - name: Test c_bindings builds + if: needs.ci-config.outputs.run_all == 'true' || contains(github.event.pull_request.labels.*.name, 'c-bindings') + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-c-bindings + # --- Label-gated: cfg-flags --- + - name: Test cfg-flag builds + if: needs.ci-config.outputs.run_all == 'true' || contains(github.event.pull_request.labels.*.name, 'cfg-flags') + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh test-cfg-flags + # --- Verify all steps ran (only when run_all) --- + - name: Verify all CI steps ran + if: needs.ci-config.outputs.run_all == 'true' + shell: bash + run: ./ci/ci-tests.sh --verify-complete coverage: - needs: fuzz + needs: [ci-config, fuzz] + if: needs.ci-config.outputs.run_all == 'true' strategy: fail-fast: false runs-on: self-hosted @@ -131,6 +223,8 @@ jobs: ./codecov --verbose upload-process --disable-search --fail-on-error -f fuzz-codecov.json -t "f421b687-4dc2-4387-ac3d-dc3b2528af57" -F 'fuzzing' benchmark: + needs: ci-config + if: needs.ci-config.outputs.run_all == 'true' runs-on: ubuntu-latest env: TOOLCHAIN: stable @@ -241,6 +335,8 @@ jobs: run: ci/check-docsrs.sh fuzz: + needs: ci-config + if: needs.ci-config.outputs.run_all == 'true' runs-on: self-hosted env: TOOLCHAIN: 1.75 diff --git a/ci/ci-tests.sh b/ci/ci-tests.sh index 820935f9100..49c58edeb25 100755 --- a/ci/ci-tests.sh +++ b/ci/ci-tests.sh @@ -24,30 +24,89 @@ PIN_RELEASE_DEPS # pin the release dependencies in our main workspace export RUST_BACKTRACE=1 -echo -e "\n\nChecking the workspace, except lightning-transaction-sync." -cargo check --quiet --color always +# All steps in order, matching the original script flow +ALL_STEPS=" +check-workspace +test-workspace +test-ldk-upgrade +check-workspace-members +test-lightning-dnssec +test-lightning-block-sync +check-lightning-transaction-sync +test-lightning-transaction-sync +test-lightning-persister +test-lightning-custom-message +test-lightning-backtrace +test-no-std +test-c-bindings +test-crate-specific +check-no-std +check-msrv-no-dev-deps +build-no-std-arm +test-cfg-flags +" + +# If a step name is passed, run just that step. Otherwise run all. +if [ -n "$1" ]; then + STEPS_TO_RUN="$1" +else + STEPS_TO_RUN="$ALL_STEPS" +fi WORKSPACE_MEMBERS=( $(cat Cargo.toml | tr '\n' '\r' | sed 's/\r //g' | tr '\r' '\n' | grep '^members =' | sed 's/members.*=.*\[//' | tr -d '"' | tr ',' ' ') ) +# Verify that all steps were executed (called at the end of CI) +if [ "$1" = "--verify-complete" ]; then + MISSING_STEPS="" + for STEP in $ALL_STEPS; do + if [[ ! " $CI_COMPLETED_STEPS " == *" $STEP "* ]]; then + MISSING_STEPS="$MISSING_STEPS $STEP" + fi + done + if [ -n "$MISSING_STEPS" ]; then + echo "ERROR: The following CI steps were not executed:$MISSING_STEPS" + exit 1 + fi + echo "All CI steps were executed successfully." + exit 0 +fi + +for STEP in $STEPS_TO_RUN; do +case "$STEP" in + +check-workspace) +echo -e "\n\nChecking the workspace, except lightning-transaction-sync." +cargo check --quiet --color always +;; + +test-workspace) echo -e "\n\nTesting the workspace, except lightning-transaction-sync." cargo test --quiet --color always +;; +test-ldk-upgrade) echo -e "\n\nTesting upgrade from prior versions of LDK" pushd lightning-tests cargo test --quiet popd +;; +check-workspace-members) echo -e "\n\nChecking and building docs for all workspace members individually..." for DIR in "${WORKSPACE_MEMBERS[@]}"; do cargo check -p "$DIR" --quiet --color always cargo doc -p "$DIR" --quiet --document-private-items done +;; +test-lightning-dnssec) echo -e "\n\nChecking and testing lightning with features" cargo test -p lightning --quiet --color always --features dnssec cargo check -p lightning --quiet --color always --features dnssec cargo doc -p lightning --quiet --document-private-items --features dnssec +;; +test-lightning-block-sync) echo -e "\n\nChecking and testing Block Sync Clients with features" cargo test -p lightning-block-sync --quiet --color always --features rest-client @@ -58,13 +117,17 @@ cargo test -p lightning-block-sync --quiet --color always --features rpc-client, cargo check -p lightning-block-sync --quiet --color always --features rpc-client,rest-client cargo test -p lightning-block-sync --quiet --color always --features rpc-client,rest-client,tokio cargo check -p lightning-block-sync --quiet --color always --features rpc-client,rest-client,tokio +;; +check-lightning-transaction-sync) echo -e "\n\nChecking Transaction Sync Clients with features." cargo check -p lightning-transaction-sync --quiet --color always --features esplora-blocking cargo check -p lightning-transaction-sync --quiet --color always --features esplora-async cargo check -p lightning-transaction-sync --quiet --color always --features esplora-async-https cargo check -p lightning-transaction-sync --quiet --color always --features electrum +;; +test-lightning-transaction-sync) if [ -z "$CI_ENV" ] && [[ -z "$BITCOIND_EXE" || -z "$ELECTRS_EXE" ]]; then echo -e "\n\nSkipping testing Transaction Sync Clients due to BITCOIND_EXE or ELECTRS_EXE being unset." cargo check -p lightning-transaction-sync --tests @@ -75,19 +138,27 @@ else cargo test -p lightning-transaction-sync --quiet --color always --features esplora-async-https cargo test -p lightning-transaction-sync --quiet --color always --features electrum fi +;; +test-lightning-persister) echo -e "\n\nChecking and testing lightning-persister with features" cargo test -p lightning-persister --quiet --color always --features tokio cargo check -p lightning-persister --quiet --color always --features tokio cargo doc -p lightning-persister --quiet --document-private-items --features tokio +;; +test-lightning-custom-message) echo -e "\n\nTest Custom Message Macros" cargo test -p lightning-custom-message --quiet --color always [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean +;; +test-lightning-backtrace) echo -e "\n\nTest backtrace-debug builds" cargo test -p lightning --quiet --color always --features backtrace +;; +test-no-std) echo -e "\n\nTesting no_std builds" for DIR in lightning-invoice lightning-rapid-gossip-sync lightning-liquidity; do cargo test -p $DIR --quiet --color always --no-default-features @@ -95,7 +166,9 @@ done cargo test -p lightning --quiet --color always --no-default-features cargo test -p lightning-background-processor --quiet --color always --no-default-features +;; +test-c-bindings) echo -e "\n\nTesting c_bindings builds" # Note that because `$RUSTFLAGS` is not passed through to doctest builds we cannot selectively # disable doctests in `c_bindings` so we skip doctests entirely here. @@ -110,35 +183,45 @@ done # disable doctests in `c_bindings` so we skip doctests entirely here. RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test -p lightning-background-processor --quiet --color always --no-default-features --lib --bins --tests RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test -p lightning --quiet --color always --no-default-features --lib --bins --tests +;; +test-crate-specific) echo -e "\n\nTesting other crate-specific builds" # Note that outbound_commitment_test only runs in this mode because of hardcoded signature values RUSTFLAGS="$RUSTFLAGS --cfg=ldk_test_vectors" cargo test -p lightning --quiet --color always --no-default-features --features=std # This one only works for lightning-invoice # check that compile with no_std and serde works in lightning-invoice cargo test -p lightning-invoice --quiet --color always --no-default-features --features serde +;; +check-no-std) echo -e "\n\nTesting no_std build on a downstream no-std crate" # check no-std compatibility across dependencies pushd no-std-check cargo check --quiet --color always [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean popd +;; +check-msrv-no-dev-deps) # Test that we can build downstream code with only the "release pins". pushd msrv-no-dev-deps-check PIN_RELEASE_DEPS cargo check --quiet [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean popd +;; +build-no-std-arm) if [ -f "$(which arm-none-eabi-gcc)" ]; then pushd no-std-check cargo build --quiet --target=thumbv7m-none-eabi [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean popd fi +;; +test-cfg-flags) echo -e "\n\nTest cfg-flag builds" RUSTFLAGS="--cfg=taproot" cargo test --quiet --color always -p lightning [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean @@ -147,3 +230,17 @@ RUSTFLAGS="--cfg=simple_close" cargo test --quiet --color always -p lightning RUSTFLAGS="--cfg=lsps1_service" cargo test --quiet --color always -p lightning-liquidity [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean RUSTFLAGS="--cfg=peer_storage" cargo test --quiet --color always -p lightning +;; + +*) +echo "Unknown step: $STEP" +exit 1 +;; + +esac + +# Log the completed step to GITHUB_ENV for the verification step +if [ -n "$GITHUB_ENV" ]; then + echo "CI_COMPLETED_STEPS=${CI_COMPLETED_STEPS:-} $STEP" >> "$GITHUB_ENV" +fi +done