From 8cd166073e86f4b270f7ba1bc2b606d575fa0850 Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Tue, 17 Feb 2026 15:50:05 +0100 Subject: [PATCH 1/3] Move shellcheck step from build job to linting job Shellcheck is a static analysis tool for shell scripts and belongs with the other linting checks rather than in the build matrix. This also prepares for splitting the build job into parallel sub-jobs, where running shellcheck in each sub-job would be redundant. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/build.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0fbc9eded5e..abc580baf13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,11 +60,6 @@ jobs: if: "matrix.platform == 'self-hosted'" run: | rustup target add thumbv7m-none-eabi - - name: shellcheck the CI and `contrib` scripts - if: "matrix.platform == 'self-hosted'" - run: | - shellcheck ci/*.sh -aP ci - shellcheck contrib/*.sh -aP contrib - name: Set RUSTFLAGS to deny warnings if: "matrix.toolchain == '1.75.0'" run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV" @@ -305,6 +300,10 @@ jobs: - name: Install clippy run: | rustup component add clippy + - name: shellcheck the CI and `contrib` scripts + run: | + shellcheck ci/*.sh -aP ci + shellcheck contrib/*.sh -aP contrib - name: Run default clippy linting run: | ./ci/check-lint.sh From 44d2563158849ef52221989a92cd96a12ef429c2 Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Tue, 17 Feb 2026 15:39:30 +0100 Subject: [PATCH 2/3] Split CI test script into parallel jobs Split the monolithic ci-tests.sh into four focused scripts that run as separate parallel jobs via a reusable workflow (ci-build.yml): - ci-tests-workspace.sh: workspace checks, tests, docs, and features - ci-tests-bindings.sh: c_bindings builds and tests - ci-tests-nostd-and-cfg.sh: no_std, crate-specific, and cfg-flag builds - ci-tests-sync.sh: block sync and transaction sync clients Shared setup (MSRV pins, backtrace) is extracted into ci-tests-common.sh. The ci-tests.sh script now delegates to the sub-scripts for local use. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/build.yml | 90 +++++--------------- .github/workflows/ci-build.yml | 88 +++++++++++++++++++ ci/ci-tests-bindings.sh | 20 +++++ ci/ci-tests-common.sh | 23 +++++ ci/ci-tests-nostd-and-cfg.sh | 50 +++++++++++ ci/ci-tests-sync.sh | 33 ++++++++ ci/ci-tests-workspace.sh | 42 ++++++++++ ci/ci-tests.sh | 149 ++------------------------------- 8 files changed, 286 insertions(+), 209 deletions(-) create mode 100644 .github/workflows/ci-build.yml create mode 100755 ci/ci-tests-bindings.sh create mode 100755 ci/ci-tests-common.sh create mode 100755 ci/ci-tests-nostd-and-cfg.sh create mode 100755 ci/ci-tests-sync.sh create mode 100755 ci/ci-tests-workspace.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index abc580baf13..b18cde52f78 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,72 +26,28 @@ jobs: cd ext-functional-test-demo cargo test --verbose --color always cargo test --verbose --color always --features test-broken - build: - strategy: - fail-fast: false - matrix: - platform: >- - ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' - && fromJSON('["self-hosted","windows-latest","macos-latest"]') - || fromJSON('["self-hosted"]') }} - toolchain: >- - ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' - && fromJSON('["stable","beta","1.75.0"]') - || fromJSON('["1.75.0"]') }} - exclude: - - platform: windows-latest - toolchain: 1.75.0 - - platform: windows-latest - toolchain: beta - - platform: macos-latest - toolchain: beta - runs-on: ${{ matrix.platform }} - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - name: Install Rust ${{ matrix.toolchain }} toolchain - run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ matrix.toolchain }} - - name: Use rust-lld linker on Windows - if: matrix.platform == 'windows-latest' - shell: bash - run: echo "RUSTFLAGS=-C linker=rust-lld" >> "$GITHUB_ENV" - - name: Install no-std-check dependencies for ARM Embedded - if: "matrix.platform == 'self-hosted'" - run: | - rustup target add thumbv7m-none-eabi - - name: Set RUSTFLAGS to deny warnings - if: "matrix.toolchain == '1.75.0'" - run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV" - - name: Enable caching for bitcoind - if: matrix.platform != 'windows-latest' - id: cache-bitcoind - uses: actions/cache@v4 - with: - path: bin/bitcoind-${{ runner.os }}-${{ runner.arch }} - key: bitcoind-${{ runner.os }}-${{ runner.arch }} - - name: Enable caching for electrs - if: matrix.platform != 'windows-latest' - id: cache-electrs - uses: actions/cache@v4 - with: - path: bin/electrs-${{ runner.os }}-${{ runner.arch }} - key: electrs-${{ runner.os }}-${{ runner.arch }} - - name: Download bitcoind/electrs - if: "matrix.platform != 'windows-latest' && (steps.cache-bitcoind.outputs.cache-hit != 'true' || steps.cache-electrs.outputs.cache-hit != 'true')" - run: | - source ./contrib/download_bitcoind_electrs.sh - mkdir bin - mv "$BITCOIND_EXE" bin/bitcoind-${{ runner.os }}-${{ runner.arch }} - mv "$ELECTRS_EXE" bin/electrs-${{ runner.os }}-${{ runner.arch }} - - name: Set bitcoind/electrs environment variables - if: matrix.platform != 'windows-latest' - run: | - echo "BITCOIND_EXE=$( pwd )/bin/bitcoind-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV" - echo "ELECTRS_EXE=$( pwd )/bin/electrs-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV" - - name: Run CI script - shell: bash # Default on Winblows is powershell - run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh + + build-workspace: + uses: ./.github/workflows/ci-build.yml + with: + script: ci/ci-tests-workspace.sh + + build-bindings: + uses: ./.github/workflows/ci-build.yml + with: + script: ci/ci-tests-bindings.sh + + build-nostd-and-cfg: + uses: ./.github/workflows/ci-build.yml + with: + script: ci/ci-tests-nostd-and-cfg.sh + install-arm-target: true + + build-sync: + uses: ./.github/workflows/ci-build.yml + with: + script: ci/ci-tests-sync.sh + setup-bitcoind: true coverage: needs: fuzz @@ -343,7 +299,7 @@ jobs: TOR_PROXY="127.0.0.1:9050" RUSTFLAGS="--cfg=tor" cargo test --verbose --color always -p lightning-net-tokio notify-failure: - needs: [build, fuzz, linting, rustfmt, check_release, check_docs, benchmark, ext-test, tor-connect, coverage] + needs: [build-workspace, build-bindings, build-nostd-and-cfg, build-sync, fuzz, linting, rustfmt, check_release, check_docs, benchmark, ext-test, tor-connect, coverage] if: failure() && github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml new file mode 100644 index 00000000000..56831292467 --- /dev/null +++ b/.github/workflows/ci-build.yml @@ -0,0 +1,88 @@ +name: CI Build Job + +on: + workflow_call: + inputs: + script: + description: CI script to run (relative to repo root) + required: true + type: string + install-arm-target: + description: Whether to install ARM embedded target + type: boolean + default: false + setup-bitcoind: + description: Whether to set up bitcoind/electrs + type: boolean + default: false + +jobs: + build: + strategy: + fail-fast: false + matrix: + platform: >- + ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' + && fromJSON('["self-hosted","windows-latest","macos-latest"]') + || fromJSON('["self-hosted"]') }} + toolchain: >- + ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' + && fromJSON('["stable","beta","1.75.0"]') + || fromJSON('["1.75.0"]') }} + exclude: + - platform: windows-latest + toolchain: 1.75.0 + - platform: windows-latest + toolchain: beta + - platform: macos-latest + toolchain: beta + runs-on: ${{ matrix.platform }} + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Install Rust ${{ matrix.toolchain }} toolchain + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ matrix.toolchain }} + - name: Use rust-lld linker on Windows + if: matrix.platform == 'windows-latest' + shell: bash + run: echo "RUSTFLAGS=-C linker=rust-lld" >> "$GITHUB_ENV" + - name: Set RUSTFLAGS to deny warnings + if: "matrix.toolchain == '1.75.0'" + run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV" + - name: Install no-std-check dependencies for ARM Embedded + if: inputs.install-arm-target && matrix.platform == 'self-hosted' + run: | + rustup target add thumbv7m-none-eabi + - name: Enable caching for bitcoind + if: inputs.setup-bitcoind && matrix.platform != 'windows-latest' + id: cache-bitcoind + uses: actions/cache@v4 + with: + path: bin/bitcoind-${{ runner.os }}-${{ runner.arch }} + key: bitcoind-${{ runner.os }}-${{ runner.arch }} + - name: Enable caching for electrs + if: inputs.setup-bitcoind && matrix.platform != 'windows-latest' + id: cache-electrs + uses: actions/cache@v4 + with: + path: bin/electrs-${{ runner.os }}-${{ runner.arch }} + key: electrs-${{ runner.os }}-${{ runner.arch }} + - name: Download bitcoind/electrs + if: >- + inputs.setup-bitcoind && matrix.platform != 'windows-latest' + && (steps.cache-bitcoind.outputs.cache-hit != 'true' + || steps.cache-electrs.outputs.cache-hit != 'true') + run: | + source ./contrib/download_bitcoind_electrs.sh + mkdir bin + mv "$BITCOIND_EXE" bin/bitcoind-${{ runner.os }}-${{ runner.arch }} + mv "$ELECTRS_EXE" bin/electrs-${{ runner.os }}-${{ runner.arch }} + - name: Set bitcoind/electrs environment variables + if: inputs.setup-bitcoind && matrix.platform != 'windows-latest' + run: | + echo "BITCOIND_EXE=$( pwd )/bin/bitcoind-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV" + echo "ELECTRS_EXE=$( pwd )/bin/electrs-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV" + - name: Run CI script + shell: bash + run: CI_ENV=1 CI_MINIMIZE_DISK_USAGE=1 ./${{ inputs.script }} diff --git a/ci/ci-tests-bindings.sh b/ci/ci-tests-bindings.sh new file mode 100755 index 00000000000..74b471a391b --- /dev/null +++ b/ci/ci-tests-bindings.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -eox pipefail + +# shellcheck source=ci/ci-tests-common.sh +source "$(dirname "$0")/ci-tests-common.sh" + +echo -e "\n\nTesting c_bindings builds" +# Note that because `$RUSTFLAGS` is not passed through to doctest builds we cannot selectively +# disable doctests in `c_bindings` so we skip doctests entirely here. +RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test --quiet --color always --lib --bins --tests + +for DIR in lightning-invoice lightning-rapid-gossip-sync; do + # check if there is a conflict between no_std and the c_bindings cfg + RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test -p $DIR --quiet --color always --no-default-features +done + +# Note that because `$RUSTFLAGS` is not passed through to doctest builds we cannot selectively +# disable doctests in `c_bindings` so we skip doctests entirely here. +RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test -p lightning-background-processor --quiet --color always --no-default-features --lib --bins --tests +RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test -p lightning --quiet --color always --no-default-features --lib --bins --tests diff --git a/ci/ci-tests-common.sh b/ci/ci-tests-common.sh new file mode 100755 index 00000000000..d60c9d07df0 --- /dev/null +++ b/ci/ci-tests-common.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# ci/ci-tests-common.sh - Shared helpers for CI test scripts. +# Source this file; do not execute it directly. +# shellcheck disable=SC2002,SC2207 + +RUSTC_MINOR_VERSION=$(rustc --version | awk '{ split($2,a,"."); print a[2] }') + +# Some crates require pinning to meet our MSRV even for our downstream users, +# which we do here. +# Further crates which appear only as dev-dependencies are pinned further down. +function PIN_RELEASE_DEPS { + return 0 # Don't fail the script if our rustc is higher than the last check +} + +PIN_RELEASE_DEPS # pin the release dependencies in our main workspace + +# The backtrace v0.3.75 crate relies on rustc 1.82 +[ "$RUSTC_MINOR_VERSION" -lt 82 ] && cargo update -p backtrace --precise "0.3.74" --quiet + +# Starting with version 1.2.0, the `idna_adapter` crate has an MSRV of rustc 1.81.0. +[ "$RUSTC_MINOR_VERSION" -lt 81 ] && cargo update -p idna_adapter --precise "1.1.0" --quiet + +export RUST_BACKTRACE=1 diff --git a/ci/ci-tests-nostd-and-cfg.sh b/ci/ci-tests-nostd-and-cfg.sh new file mode 100755 index 00000000000..937ab91a32c --- /dev/null +++ b/ci/ci-tests-nostd-and-cfg.sh @@ -0,0 +1,50 @@ +#!/bin/bash +set -eox pipefail + +# shellcheck source=ci/ci-tests-common.sh +source "$(dirname "$0")/ci-tests-common.sh" + +echo -e "\n\nTesting no_std builds" +for DIR in lightning-invoice lightning-rapid-gossip-sync lightning-liquidity; do + cargo test -p $DIR --quiet --color always --no-default-features +done + +cargo test -p lightning --quiet --color always --no-default-features +cargo test -p lightning-background-processor --quiet --color always --no-default-features + +echo -e "\n\nTesting other crate-specific builds" +# Note that outbound_commitment_test only runs in this mode because of hardcoded signature values +RUSTFLAGS="$RUSTFLAGS --cfg=ldk_test_vectors" cargo test -p lightning --quiet --color always --no-default-features --features=std +# This one only works for lightning-invoice +# check that compile with no_std and serde works in lightning-invoice +cargo test -p lightning-invoice --quiet --color always --no-default-features --features serde + +echo -e "\n\nTesting no_std build on a downstream no-std crate" +# check no-std compatibility across dependencies +pushd no-std-check +cargo check --quiet --color always +[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean +popd + +# Test that we can build downstream code with only the "release pins". +pushd msrv-no-dev-deps-check +PIN_RELEASE_DEPS +cargo check --quiet +[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean +popd + +if [ -f "$(which arm-none-eabi-gcc)" ]; then + pushd no-std-check + cargo build --quiet --target=thumbv7m-none-eabi + [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean + popd +fi + +echo -e "\n\nTest cfg-flag builds" +RUSTFLAGS="--cfg=taproot" cargo test --quiet --color always -p lightning +[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean +RUSTFLAGS="--cfg=simple_close" cargo test --quiet --color always -p lightning +[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean +RUSTFLAGS="--cfg=lsps1_service" cargo test --quiet --color always -p lightning-liquidity +[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean +RUSTFLAGS="--cfg=peer_storage" cargo test --quiet --color always -p lightning diff --git a/ci/ci-tests-sync.sh b/ci/ci-tests-sync.sh new file mode 100755 index 00000000000..ef836a60413 --- /dev/null +++ b/ci/ci-tests-sync.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -eox pipefail + +# shellcheck source=ci/ci-tests-common.sh +source "$(dirname "$0")/ci-tests-common.sh" + +echo -e "\n\nChecking and testing Block Sync Clients with features" + +cargo test -p lightning-block-sync --quiet --color always --features rest-client +cargo check -p lightning-block-sync --quiet --color always --features rest-client +cargo test -p lightning-block-sync --quiet --color always --features rpc-client +cargo check -p lightning-block-sync --quiet --color always --features rpc-client +cargo test -p lightning-block-sync --quiet --color always --features rpc-client,rest-client +cargo check -p lightning-block-sync --quiet --color always --features rpc-client,rest-client +cargo test -p lightning-block-sync --quiet --color always --features rpc-client,rest-client,tokio +cargo check -p lightning-block-sync --quiet --color always --features rpc-client,rest-client,tokio + +echo -e "\n\nChecking Transaction Sync Clients with features." +cargo check -p lightning-transaction-sync --quiet --color always --features esplora-blocking +cargo check -p lightning-transaction-sync --quiet --color always --features esplora-async +cargo check -p lightning-transaction-sync --quiet --color always --features esplora-async-https +cargo check -p lightning-transaction-sync --quiet --color always --features electrum + +if [ -z "$CI_ENV" ] && [[ -z "$BITCOIND_EXE" || -z "$ELECTRS_EXE" ]]; then + echo -e "\n\nSkipping testing Transaction Sync Clients due to BITCOIND_EXE or ELECTRS_EXE being unset." + cargo check -p lightning-transaction-sync --tests +else + echo -e "\n\nTesting Transaction Sync Clients with features." + cargo test -p lightning-transaction-sync --quiet --color always --features esplora-blocking + cargo test -p lightning-transaction-sync --quiet --color always --features esplora-async + cargo test -p lightning-transaction-sync --quiet --color always --features esplora-async-https + cargo test -p lightning-transaction-sync --quiet --color always --features electrum +fi diff --git a/ci/ci-tests-workspace.sh b/ci/ci-tests-workspace.sh new file mode 100755 index 00000000000..9ac13004622 --- /dev/null +++ b/ci/ci-tests-workspace.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#shellcheck disable=SC2002,SC2207 +set -eox pipefail + +# shellcheck source=ci/ci-tests-common.sh +source "$(dirname "$0")/ci-tests-common.sh" + +echo -e "\n\nChecking the workspace, except lightning-transaction-sync." +cargo check --quiet --color always + +WORKSPACE_MEMBERS=( $(cat Cargo.toml | tr '\n' '\r' | sed 's/\r //g' | tr '\r' '\n' | grep '^members =' | sed 's/members.*=.*\[//' | tr -d '"' | tr ',' ' ') ) + +echo -e "\n\nTesting the workspace, except lightning-transaction-sync." +cargo test --quiet --color always + +echo -e "\n\nTesting upgrade from prior versions of LDK" +pushd lightning-tests +cargo test --quiet +popd + +echo -e "\n\nChecking and building docs for all workspace members individually..." +for DIR in "${WORKSPACE_MEMBERS[@]}"; do + cargo check -p "$DIR" --quiet --color always + cargo doc -p "$DIR" --quiet --document-private-items +done + +echo -e "\n\nChecking and testing lightning with features" +cargo test -p lightning --quiet --color always --features dnssec +cargo check -p lightning --quiet --color always --features dnssec +cargo doc -p lightning --quiet --document-private-items --features dnssec + +echo -e "\n\nChecking and testing lightning-persister with features" +cargo test -p lightning-persister --quiet --color always --features tokio +cargo check -p lightning-persister --quiet --color always --features tokio +cargo doc -p lightning-persister --quiet --document-private-items --features tokio + +echo -e "\n\nTest Custom Message Macros" +cargo test -p lightning-custom-message --quiet --color always +[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean + +echo -e "\n\nTest backtrace-debug builds" +cargo test -p lightning --quiet --color always --features backtrace diff --git a/ci/ci-tests.sh b/ci/ci-tests.sh index 83b2af277f5..a5478e4bba1 100755 --- a/ci/ci-tests.sh +++ b/ci/ci-tests.sh @@ -1,146 +1,11 @@ #!/bin/bash -#shellcheck disable=SC2002,SC2207 set -eox pipefail -RUSTC_MINOR_VERSION=$(rustc --version | awk '{ split($2,a,"."); print a[2] }') +# Run all CI test groups sequentially for local testing. +# In GitHub Actions, these run as separate parallel jobs. -# Some crates require pinning to meet our MSRV even for our downstream users, -# which we do here. -# Further crates which appear only as dev-dependencies are pinned further down. -function PIN_RELEASE_DEPS { - return 0 # Don't fail the script if our rustc is higher than the last check -} - -PIN_RELEASE_DEPS # pin the release dependencies in our main workspace - -# The backtrace v0.3.75 crate relies on rustc 1.82 -[ "$RUSTC_MINOR_VERSION" -lt 82 ] && cargo update -p backtrace --precise "0.3.74" --quiet - -# Starting with version 1.2.0, the `idna_adapter` crate has an MSRV of rustc 1.81.0. -[ "$RUSTC_MINOR_VERSION" -lt 81 ] && cargo update -p idna_adapter --precise "1.1.0" --quiet - -export RUST_BACKTRACE=1 - -echo -e "\n\nChecking the workspace, except lightning-transaction-sync." -cargo check --quiet --color always - -WORKSPACE_MEMBERS=( $(cat Cargo.toml | tr '\n' '\r' | sed 's/\r //g' | tr '\r' '\n' | grep '^members =' | sed 's/members.*=.*\[//' | tr -d '"' | tr ',' ' ') ) - -echo -e "\n\nTesting the workspace, except lightning-transaction-sync." -cargo test --quiet --color always - -echo -e "\n\nTesting upgrade from prior versions of LDK" -pushd lightning-tests -cargo test --quiet -popd - -echo -e "\n\nChecking and building docs for all workspace members individually..." -for DIR in "${WORKSPACE_MEMBERS[@]}"; do - cargo check -p "$DIR" --quiet --color always - cargo doc -p "$DIR" --quiet --document-private-items -done - -echo -e "\n\nChecking and testing lightning with features" -cargo test -p lightning --quiet --color always --features dnssec -cargo check -p lightning --quiet --color always --features dnssec -cargo doc -p lightning --quiet --document-private-items --features dnssec - -echo -e "\n\nChecking and testing Block Sync Clients with features" - -cargo test -p lightning-block-sync --quiet --color always --features rest-client -cargo check -p lightning-block-sync --quiet --color always --features rest-client -cargo test -p lightning-block-sync --quiet --color always --features rpc-client -cargo check -p lightning-block-sync --quiet --color always --features rpc-client -cargo test -p lightning-block-sync --quiet --color always --features rpc-client,rest-client -cargo check -p lightning-block-sync --quiet --color always --features rpc-client,rest-client -cargo test -p lightning-block-sync --quiet --color always --features rpc-client,rest-client,tokio -cargo check -p lightning-block-sync --quiet --color always --features rpc-client,rest-client,tokio - -echo -e "\n\nChecking Transaction Sync Clients with features." -cargo check -p lightning-transaction-sync --quiet --color always --features esplora-blocking -cargo check -p lightning-transaction-sync --quiet --color always --features esplora-async -cargo check -p lightning-transaction-sync --quiet --color always --features esplora-async-https -cargo check -p lightning-transaction-sync --quiet --color always --features electrum - -if [ -z "$CI_ENV" ] && [[ -z "$BITCOIND_EXE" || -z "$ELECTRS_EXE" ]]; then - echo -e "\n\nSkipping testing Transaction Sync Clients due to BITCOIND_EXE or ELECTRS_EXE being unset." - cargo check -p lightning-transaction-sync --tests -else - echo -e "\n\nTesting Transaction Sync Clients with features." - cargo test -p lightning-transaction-sync --quiet --color always --features esplora-blocking - cargo test -p lightning-transaction-sync --quiet --color always --features esplora-async - cargo test -p lightning-transaction-sync --quiet --color always --features esplora-async-https - cargo test -p lightning-transaction-sync --quiet --color always --features electrum -fi - -echo -e "\n\nChecking and testing lightning-persister with features" -cargo test -p lightning-persister --quiet --color always --features tokio -cargo check -p lightning-persister --quiet --color always --features tokio -cargo doc -p lightning-persister --quiet --document-private-items --features tokio - -echo -e "\n\nTest Custom Message Macros" -cargo test -p lightning-custom-message --quiet --color always -[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean - -echo -e "\n\nTest backtrace-debug builds" -cargo test -p lightning --quiet --color always --features backtrace - -echo -e "\n\nTesting no_std builds" -for DIR in lightning-invoice lightning-rapid-gossip-sync lightning-liquidity; do - cargo test -p $DIR --quiet --color always --no-default-features -done - -cargo test -p lightning --quiet --color always --no-default-features -cargo test -p lightning-background-processor --quiet --color always --no-default-features - -echo -e "\n\nTesting c_bindings builds" -# Note that because `$RUSTFLAGS` is not passed through to doctest builds we cannot selectively -# disable doctests in `c_bindings` so we skip doctests entirely here. -RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test --quiet --color always --lib --bins --tests - -for DIR in lightning-invoice lightning-rapid-gossip-sync; do - # check if there is a conflict between no_std and the c_bindings cfg - RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test -p $DIR --quiet --color always --no-default-features -done - -# Note that because `$RUSTFLAGS` is not passed through to doctest builds we cannot selectively -# disable doctests in `c_bindings` so we skip doctests entirely here. -RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test -p lightning-background-processor --quiet --color always --no-default-features --lib --bins --tests -RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test -p lightning --quiet --color always --no-default-features --lib --bins --tests - -echo -e "\n\nTesting other crate-specific builds" -# Note that outbound_commitment_test only runs in this mode because of hardcoded signature values -RUSTFLAGS="$RUSTFLAGS --cfg=ldk_test_vectors" cargo test -p lightning --quiet --color always --no-default-features --features=std -# This one only works for lightning-invoice -# check that compile with no_std and serde works in lightning-invoice -cargo test -p lightning-invoice --quiet --color always --no-default-features --features serde - -echo -e "\n\nTesting no_std build on a downstream no-std crate" -# check no-std compatibility across dependencies -pushd no-std-check -cargo check --quiet --color always -[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean -popd - -# Test that we can build downstream code with only the "release pins". -pushd msrv-no-dev-deps-check -PIN_RELEASE_DEPS -cargo check --quiet -[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean -popd - -if [ -f "$(which arm-none-eabi-gcc)" ]; then - pushd no-std-check - cargo build --quiet --target=thumbv7m-none-eabi - [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean - popd -fi - -echo -e "\n\nTest cfg-flag builds" -RUSTFLAGS="--cfg=taproot" cargo test --quiet --color always -p lightning -[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean -RUSTFLAGS="--cfg=simple_close" cargo test --quiet --color always -p lightning -[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean -RUSTFLAGS="--cfg=lsps1_service" cargo test --quiet --color always -p lightning-liquidity -[ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean -RUSTFLAGS="--cfg=peer_storage" cargo test --quiet --color always -p lightning +DIR="$(dirname "$0")" +"$DIR/ci-tests-workspace.sh" +"$DIR/ci-tests-bindings.sh" +"$DIR/ci-tests-nostd-and-cfg.sh" +"$DIR/ci-tests-sync.sh" From 55cda1745b385b01c424a724524e98e91939188e Mon Sep 17 00:00:00 2001 From: Joost Jager Date: Tue, 17 Feb 2026 17:31:49 +0100 Subject: [PATCH 3/3] [DROP ME] Add per-step timing instrumentation to CI test scripts Add ci_step_start/ci_step_end helpers to ci-tests-common.sh that track elapsed time per logical block and emit GitHub Actions ::group::/::endgroup:: markers for collapsible log sections. Each job prints a timing summary at the end for easy comparison when rebalancing work across the parallel jobs. Co-Authored-By: Claude Opus 4.6 --- ci/ci-tests-bindings.sh | 7 ++++++- ci/ci-tests-common.sh | 26 ++++++++++++++++++++++++++ ci/ci-tests-nostd-and-cfg.sh | 27 +++++++++++++++++++++++---- ci/ci-tests-sync.sh | 33 ++++++++++++++++++++++++++++----- ci/ci-tests-workspace.sh | 26 ++++++++++++++++++-------- 5 files changed, 101 insertions(+), 18 deletions(-) diff --git a/ci/ci-tests-bindings.sh b/ci/ci-tests-bindings.sh index 74b471a391b..43534a4610a 100755 --- a/ci/ci-tests-bindings.sh +++ b/ci/ci-tests-bindings.sh @@ -4,11 +4,13 @@ set -eox pipefail # shellcheck source=ci/ci-tests-common.sh source "$(dirname "$0")/ci-tests-common.sh" -echo -e "\n\nTesting c_bindings builds" +ci_step_start "Test c_bindings full workspace" # Note that because `$RUSTFLAGS` is not passed through to doctest builds we cannot selectively # disable doctests in `c_bindings` so we skip doctests entirely here. RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test --quiet --color always --lib --bins --tests +ci_step_end +ci_step_start "Test c_bindings no-default-features per crate" for DIR in lightning-invoice lightning-rapid-gossip-sync; do # check if there is a conflict between no_std and the c_bindings cfg RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test -p $DIR --quiet --color always --no-default-features @@ -18,3 +20,6 @@ done # disable doctests in `c_bindings` so we skip doctests entirely here. RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test -p lightning-background-processor --quiet --color always --no-default-features --lib --bins --tests RUSTFLAGS="$RUSTFLAGS --cfg=c_bindings" cargo test -p lightning --quiet --color always --no-default-features --lib --bins --tests +ci_step_end + +ci_print_summary diff --git a/ci/ci-tests-common.sh b/ci/ci-tests-common.sh index d60c9d07df0..d3b2cf7f2a1 100755 --- a/ci/ci-tests-common.sh +++ b/ci/ci-tests-common.sh @@ -21,3 +21,29 @@ PIN_RELEASE_DEPS # pin the release dependencies in our main workspace [ "$RUSTC_MINOR_VERSION" -lt 81 ] && cargo update -p idna_adapter --precise "1.1.0" --quiet export RUST_BACKTRACE=1 + +CI_STEP_START_TIME="" +CI_STEP_NAME="" +CI_SUMMARY="" + +function ci_step_start { + CI_STEP_NAME="$1" + CI_STEP_START_TIME=$(date +%s) + echo "::group::$CI_STEP_NAME" +} + +function ci_step_end { + local elapsed=$(( $(date +%s) - CI_STEP_START_TIME )) + local mins=$(( elapsed / 60 )) + local secs=$(( elapsed % 60 )) + echo "::endgroup::" + echo "TIMING: ${CI_STEP_NAME} completed in ${mins}m ${secs}s" + CI_SUMMARY="${CI_SUMMARY}${mins}m ${secs}s ${CI_STEP_NAME}\n" +} + +function ci_print_summary { + echo "" + echo "===== TIMING SUMMARY =====" + echo -e "$CI_SUMMARY" + echo "==========================" +} diff --git a/ci/ci-tests-nostd-and-cfg.sh b/ci/ci-tests-nostd-and-cfg.sh index 937ab91a32c..57100af3ea7 100755 --- a/ci/ci-tests-nostd-and-cfg.sh +++ b/ci/ci-tests-nostd-and-cfg.sh @@ -4,47 +4,66 @@ set -eox pipefail # shellcheck source=ci/ci-tests-common.sh source "$(dirname "$0")/ci-tests-common.sh" -echo -e "\n\nTesting no_std builds" +ci_step_start "Test no_std builds" for DIR in lightning-invoice lightning-rapid-gossip-sync lightning-liquidity; do cargo test -p $DIR --quiet --color always --no-default-features done cargo test -p lightning --quiet --color always --no-default-features cargo test -p lightning-background-processor --quiet --color always --no-default-features +ci_step_end -echo -e "\n\nTesting other crate-specific builds" +ci_step_start "Test ldk_test_vectors and serde" # Note that outbound_commitment_test only runs in this mode because of hardcoded signature values RUSTFLAGS="$RUSTFLAGS --cfg=ldk_test_vectors" cargo test -p lightning --quiet --color always --no-default-features --features=std # This one only works for lightning-invoice # check that compile with no_std and serde works in lightning-invoice cargo test -p lightning-invoice --quiet --color always --no-default-features --features serde +ci_step_end -echo -e "\n\nTesting no_std build on a downstream no-std crate" +ci_step_start "Test no-std-check downstream crate" # check no-std compatibility across dependencies pushd no-std-check cargo check --quiet --color always [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean popd +ci_step_end +ci_step_start "Test msrv-no-dev-deps-check" # Test that we can build downstream code with only the "release pins". pushd msrv-no-dev-deps-check PIN_RELEASE_DEPS cargo check --quiet [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean popd +ci_step_end if [ -f "$(which arm-none-eabi-gcc)" ]; then + ci_step_start "Build no-std-check for ARM" pushd no-std-check cargo build --quiet --target=thumbv7m-none-eabi [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean popd + ci_step_end fi -echo -e "\n\nTest cfg-flag builds" +ci_step_start "Test taproot cfg flag" RUSTFLAGS="--cfg=taproot" cargo test --quiet --color always -p lightning [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean +ci_step_end + +ci_step_start "Test simple_close cfg flag" RUSTFLAGS="--cfg=simple_close" cargo test --quiet --color always -p lightning [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean +ci_step_end + +ci_step_start "Test lsps1_service cfg flag" RUSTFLAGS="--cfg=lsps1_service" cargo test --quiet --color always -p lightning-liquidity [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean +ci_step_end + +ci_step_start "Test peer_storage cfg flag" RUSTFLAGS="--cfg=peer_storage" cargo test --quiet --color always -p lightning +ci_step_end + +ci_print_summary diff --git a/ci/ci-tests-sync.sh b/ci/ci-tests-sync.sh index ef836a60413..4dd7d015403 100755 --- a/ci/ci-tests-sync.sh +++ b/ci/ci-tests-sync.sh @@ -4,30 +4,53 @@ set -eox pipefail # shellcheck source=ci/ci-tests-common.sh source "$(dirname "$0")/ci-tests-common.sh" -echo -e "\n\nChecking and testing Block Sync Clients with features" - +ci_step_start "Test lightning-block-sync with rest-client" cargo test -p lightning-block-sync --quiet --color always --features rest-client cargo check -p lightning-block-sync --quiet --color always --features rest-client +ci_step_end + +ci_step_start "Test lightning-block-sync with rpc-client" cargo test -p lightning-block-sync --quiet --color always --features rpc-client cargo check -p lightning-block-sync --quiet --color always --features rpc-client +ci_step_end + +ci_step_start "Test lightning-block-sync with rpc-client,rest-client" cargo test -p lightning-block-sync --quiet --color always --features rpc-client,rest-client cargo check -p lightning-block-sync --quiet --color always --features rpc-client,rest-client +ci_step_end + +ci_step_start "Test lightning-block-sync with rpc-client,rest-client,tokio" cargo test -p lightning-block-sync --quiet --color always --features rpc-client,rest-client,tokio cargo check -p lightning-block-sync --quiet --color always --features rpc-client,rest-client,tokio +ci_step_end -echo -e "\n\nChecking Transaction Sync Clients with features." +ci_step_start "Check lightning-transaction-sync feature combos" cargo check -p lightning-transaction-sync --quiet --color always --features esplora-blocking cargo check -p lightning-transaction-sync --quiet --color always --features esplora-async cargo check -p lightning-transaction-sync --quiet --color always --features esplora-async-https cargo check -p lightning-transaction-sync --quiet --color always --features electrum +ci_step_end if [ -z "$CI_ENV" ] && [[ -z "$BITCOIND_EXE" || -z "$ELECTRS_EXE" ]]; then - echo -e "\n\nSkipping testing Transaction Sync Clients due to BITCOIND_EXE or ELECTRS_EXE being unset." + ci_step_start "Check lightning-transaction-sync tests (no bitcoind)" cargo check -p lightning-transaction-sync --tests + ci_step_end else - echo -e "\n\nTesting Transaction Sync Clients with features." + ci_step_start "Test lightning-transaction-sync with esplora-blocking" cargo test -p lightning-transaction-sync --quiet --color always --features esplora-blocking + ci_step_end + + ci_step_start "Test lightning-transaction-sync with esplora-async" cargo test -p lightning-transaction-sync --quiet --color always --features esplora-async + ci_step_end + + ci_step_start "Test lightning-transaction-sync with esplora-async-https" cargo test -p lightning-transaction-sync --quiet --color always --features esplora-async-https + ci_step_end + + ci_step_start "Test lightning-transaction-sync with electrum" cargo test -p lightning-transaction-sync --quiet --color always --features electrum + ci_step_end fi + +ci_print_summary diff --git a/ci/ci-tests-workspace.sh b/ci/ci-tests-workspace.sh index 9ac13004622..c22aea15471 100755 --- a/ci/ci-tests-workspace.sh +++ b/ci/ci-tests-workspace.sh @@ -5,38 +5,48 @@ set -eox pipefail # shellcheck source=ci/ci-tests-common.sh source "$(dirname "$0")/ci-tests-common.sh" -echo -e "\n\nChecking the workspace, except lightning-transaction-sync." +ci_step_start "Check workspace" cargo check --quiet --color always +ci_step_end WORKSPACE_MEMBERS=( $(cat Cargo.toml | tr '\n' '\r' | sed 's/\r //g' | tr '\r' '\n' | grep '^members =' | sed 's/members.*=.*\[//' | tr -d '"' | tr ',' ' ') ) -echo -e "\n\nTesting the workspace, except lightning-transaction-sync." +ci_step_start "Test workspace" cargo test --quiet --color always +ci_step_end -echo -e "\n\nTesting upgrade from prior versions of LDK" +ci_step_start "Test LDK upgrade compatibility" pushd lightning-tests cargo test --quiet popd +ci_step_end -echo -e "\n\nChecking and building docs for all workspace members individually..." +ci_step_start "Check and build docs for all workspace members" for DIR in "${WORKSPACE_MEMBERS[@]}"; do cargo check -p "$DIR" --quiet --color always cargo doc -p "$DIR" --quiet --document-private-items done +ci_step_end -echo -e "\n\nChecking and testing lightning with features" +ci_step_start "Test lightning with dnssec feature" cargo test -p lightning --quiet --color always --features dnssec cargo check -p lightning --quiet --color always --features dnssec cargo doc -p lightning --quiet --document-private-items --features dnssec +ci_step_end -echo -e "\n\nChecking and testing lightning-persister with features" +ci_step_start "Test lightning-persister with tokio feature" cargo test -p lightning-persister --quiet --color always --features tokio cargo check -p lightning-persister --quiet --color always --features tokio cargo doc -p lightning-persister --quiet --document-private-items --features tokio +ci_step_end -echo -e "\n\nTest Custom Message Macros" +ci_step_start "Test lightning-custom-message" cargo test -p lightning-custom-message --quiet --color always [ "$CI_MINIMIZE_DISK_USAGE" != "" ] && cargo clean +ci_step_end -echo -e "\n\nTest backtrace-debug builds" +ci_step_start "Test lightning with backtrace feature" cargo test -p lightning --quiet --color always --features backtrace +ci_step_end + +ci_print_summary