diff --git a/.github/workflows/build-cln.yaml b/.github/workflows/build-cln.yaml new file mode 100644 index 000000000000..c11365b3a0b8 --- /dev/null +++ b/.github/workflows/build-cln.yaml @@ -0,0 +1,122 @@ +--- +name: Build CLN +on: + workflow_call: + inputs: + cfg: + description: 'Configuration name (e.g., compile-gcc)' + required: true + type: string + compiler: + description: 'Compiler to use (gcc or clang)' + required: true + type: string + valgrind: + description: 'Enable valgrind (0 or 1)' + required: false + type: number + default: 1 + asan: + description: 'Enable address sanitizer (0 or 1)' + required: false + type: number + default: 0 + ubsan: + description: 'Enable undefined behavior sanitizer (0 or 1)' + required: false + type: number + default: 0 + coptflags: + description: 'Additional compiler optimization flags' + required: false + type: string + default: '' + os: + description: 'Operating system (ubuntu-22.04 or macos-14)' + required: false + type: string + default: 'ubuntu-22.04' + +env: + RUST_PROFILE: release + SLOW_MACHINE: 1 + +jobs: + build: + name: Build CLN ${{ inputs.cfg }} + runs-on: ${{ inputs.os }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies (Linux) + if: ${{ inputs.os == 'ubuntu-22.04' }} + run: | + bash -x .github/scripts/setup.sh + + - name: Install dependencies (macOS) + if: ${{ startsWith(inputs.os, 'macos-') }} + env: + GRPC_PYTHON_BUILD_SYSTEM_OPENSSL: 1 + GRPC_PYTHON_BUILD_SYSTEM_ZLIB: 1 + run: | + export PATH="/usr/local/opt:/Users/runner/.local/bin:/opt/homebrew/bin/python3.10/bin:$PATH" + brew install gnu-sed autoconf automake libtool protobuf openssl lowdown libsodium + uv sync --all-groups + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.9 + + - name: Build + env: + COMPILER: ${{ inputs.compiler }} + ASAN: ${{ inputs.asan }} + UBSAN: ${{ inputs.ubsan }} + VALGRIND: ${{ inputs.valgrind }} + COMPAT: 1 + CFG: ${{ inputs.cfg }} + RUSTC_WRAPPER: sccache + # macOS-specific environment variables + CPATH: ${{ startsWith(inputs.os, 'macos-') && '/opt/homebrew/include' || '' }} + LIBRARY_PATH: ${{ startsWith(inputs.os, 'macos-') && '/opt/homebrew/lib' || '' }} + GRPC_PYTHON_BUILD_SYSTEM_OPENSSL: ${{ startsWith(inputs.os, 'macos-') && '1' || '' }} + GRPC_PYTHON_BUILD_SYSTEM_ZLIB: ${{ startsWith(inputs.os, 'macos-') && '1' || '' }} + run: | + set -e + + # Determine configure flags based on OS + if [[ "${{ inputs.os }}" == ubuntu-* ]]; then + CONFIGURE_FLAGS="--enable-debugbuild" + CC_WRAPPER="sccache $COMPILER" + else + # macOS: disable valgrind (not available) and compat + CONFIGURE_FLAGS="--disable-valgrind --disable-compat" + CC_WRAPPER="$COMPILER" + fi + + ./configure $CONFIGURE_FLAGS CC="$CC_WRAPPER" ${{ inputs.coptflags }} + + uv run make -j $(nproc) testpack.tar.bz2 + + # Rename now so we don't clash + mv testpack.tar.bz2 cln-${CFG}.tar.bz2 + + - name: Check rust packages + env: + RUSTC_WRAPPER: sccache + run: cargo test --all + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: cln-${{ inputs.cfg }}.tar.bz2 + path: cln-${{ inputs.cfg }}.tar.bz2 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cd400b61eb26..ba2bf4c3da54 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,6 +6,7 @@ on: - "master" pull_request: types: [opened, synchronize, edited] + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -97,70 +98,48 @@ jobs: - name: Check docs run: uv run make check-doc - compile: - name: Compile CLN ${{ matrix.cfg }} - runs-on: ubuntu-22.04 - timeout-minutes: 30 + compile-gcc: + name: Compile CLN (GCC) needs: - prebuild - strategy: - fail-fast: true - matrix: - include: - - CFG: compile-gcc - VALGRIND: 1 - COMPILER: gcc - - CFG: compile-gcc-O3 - VALGRIND: 1 - COMPILER: gcc - COPTFLAGS_VAR: COPTFLAGS="-O3 -Werror" - # While we're at it let's try to compile with clang - - CFG: compile-clang - VALGRIND: 1 - COMPILER: clang - - CFG: compile-clang-sanitizers - COMPILER: clang - ASAN: 1 - UBSAN: 1 - VALGRIND: 0 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Build - env: - COMPILER: ${{ matrix.COMPILER }} - ASAN: ${{ matrix.ASAN }} - UBSAN: ${{ matrix.UBSAN }} - VALGRIND: ${{ matrix.VALGRIND }} - COMPAT: 1 - CFG: ${{ matrix.CFG }} - run: | - set -e - ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }} - - uv run make -j $(nproc) testpack.tar.bz2 - - # Rename now so we don't clash - mv testpack.tar.bz2 cln-${CFG}.tar.bz2 - - name: Check rust packages - run: cargo test --all - - uses: actions/upload-artifact@v4 - with: - name: cln-${{ matrix.CFG }}.tar.bz2 - path: cln-${{ matrix.CFG }}.tar.bz2 + uses: ./.github/workflows/build-cln.yaml + with: + cfg: compile-gcc + compiler: gcc + valgrind: 1 + + compile-gcc-O3: + name: Compile CLN (GCC -O3) + needs: + - prebuild + uses: ./.github/workflows/build-cln.yaml + with: + cfg: compile-gcc-O3 + compiler: gcc + valgrind: 1 + coptflags: 'COPTFLAGS="-O3 -Werror"' + + compile-clang: + name: Compile CLN (Clang) + needs: + - prebuild + uses: ./.github/workflows/build-cln.yaml + with: + cfg: compile-clang + compiler: clang + valgrind: 1 + + compile-clang-sanitizers: + name: Compile CLN (Clang with sanitizers) + needs: + - prebuild + uses: ./.github/workflows/build-cln.yaml + with: + cfg: compile-clang-sanitizers + compiler: clang + valgrind: 0 + asan: 1 + ubsan: 1 check-units: # The unit test checks are not in the critical path (not dependent @@ -171,7 +150,8 @@ jobs: env: BOLTDIR: bolts needs: - - compile + - compile-gcc + - compile-clang-sanitizers strategy: fail-fast: true matrix: @@ -235,329 +215,156 @@ jobs: ./configure --enable-debugbuild --enable-fuzzing --enable-address-sanitizer --enable-ub-sanitizer --disable-valgrind CC=clang uv run make -j $(nproc) check-fuzz - check-downgrade: - name: Check we can downgrade the node - runs-on: ubuntu-22.04 + check-downgrade-sqlite: + name: Check downgrade (SQLite) needs: - - compile - strategy: - fail-fast: false - matrix: - include: - - CFG: compile-gcc - TEST_DB_PROVIDER: sqlite3 - TEST_NETWORK: regtest - VALGRIND: 1 - - CFG: compile-gcc - TEST_DB_PROVIDER: postgres - TEST_NETWORK: regtest - - CFG: compile-gcc - TEST_DB_PROVIDER: sqlite3 - TEST_NETWORK: liquid-regtest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Install bitcoind - env: - TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - run: .github/scripts/install-bitcoind.sh - - - name: Download build - uses: actions/download-artifact@v4 - with: - name: cln-${{ matrix.CFG }}.tar.bz2 - - - name: Unpack pre-built CLN - env: - CFG: ${{ matrix.CFG }} - run: | - tar -xaf cln-${CFG}.tar.bz2 - - - name: Fetch and unpack previous CLN - run: | - mkdir /tmp/old-cln - cd /tmp/old-cln - wget https://github.com/ElementsProject/lightning/releases/download/v25.09/clightning-v25.09-Ubuntu-22.04-amd64.tar.xz - tar -xaf clightning-v25.09-Ubuntu-22.04-amd64.tar.xz - - - name: Switch network - if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }} - run: | - # Loading the network from config.vars rather than the envvar is a terrible idea... - sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars - cat config.vars - - - name: Test - env: - SLOW_MACHINE: 1 - PYTEST_PAR: 10 - TEST_DEBUG: 1 - TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} - TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - LIGHTNINGD_POSTGRES_NO_VACUUM: 1 - VALGRIND: ${{ matrix.VALGRIND }} - PREV_LIGHTNINGD: /tmp/old-cln/usr/bin/lightningd - run: | - env - cat config.vars - uv run eatmydata pytest tests/test_downgrade.py -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS} - - integration: - name: Test CLN ${{ matrix.name }} - runs-on: ubuntu-22.04 - timeout-minutes: 120 - env: - RUST_PROFILE: release # Has to match the one in the compile step - PYTEST_OPTS: --timeout=1200 --durations=10 + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Check downgrade (SQLite) + cfg: compile-gcc + test_path: tests/test_downgrade.py + test_db_provider: sqlite3 + valgrind: 1 + setup_old_cln: true + + check-downgrade-postgres: + name: Check downgrade (Postgres) needs: - - compile - strategy: - fail-fast: false - matrix: - include: - - NAME: gcc - CFG: compile-gcc - TEST_DB_PROVIDER: sqlite3 - COMPILER: gcc - TEST_NETWORK: regtest - # While we're at it let's try to compile with clang - - NAME: clang - CFG: compile-clang - TEST_DB_PROVIDER: sqlite3 - COMPILER: clang - TEST_NETWORK: regtest - # And of course we want to test postgres too - - NAME: postgres - CFG: compile-gcc - COMPILER: gcc - TEST_DB_PROVIDER: postgres - TEST_NETWORK: regtest - # And don't forget about elements (like cdecker did when - # reworking the CI...) - - NAME: liquid - CFG: compile-gcc - COMPILER: gcc - TEST_NETWORK: liquid-regtest - TEST_DB_PROVIDER: sqlite3 - # And dual funding! - - NAME: dual-fund - CFG: compile-gcc - TEST_DB_PROVIDER: sqlite3 - COMPILER: gcc - TEST_NETWORK: regtest - EXPERIMENTAL_DUAL_FUND: 1 - # And splicing! - - NAME: splicing - CFG: compile-gcc - TEST_DB_PROVIDER: sqlite3 - COMPILER: gcc - TEST_NETWORK: regtest - EXPERIMENTAL_SPLICING: 1 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Install bitcoind - env: - TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - run: .github/scripts/install-bitcoind.sh - - - name: Download build - uses: actions/download-artifact@v4 - with: - name: cln-${{ matrix.CFG }}.tar.bz2 - - - name: Unpack pre-built CLN - env: - CFG: ${{ matrix.CFG }} - run: | - tar -xaf cln-${CFG}.tar.bz2 - - - name: Switch network - if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }} - run: | - # Loading the network from config.vars rather than the envvar is a terrible idea... - sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars - cat config.vars - - - name: Test - env: - COMPILER: ${{ matrix.COMPILER }} - EXPERIMENTAL_DUAL_FUND: ${{ matrix.EXPERIMENTAL_DUAL_FUND }} - EXPERIMENTAL_SPLICING: ${{ matrix.EXPERIMENTAL_SPLICING }} - COMPAT: 1 - CFG: ${{ matrix.CFG }} - SLOW_MACHINE: 1 - PYTEST_PAR: 10 - TEST_DEBUG: 1 - TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }} - TEST_NETWORK: ${{ matrix.TEST_NETWORK }} - LIGHTNINGD_POSTGRES_NO_VACUUM: 1 - run: | - env - cat config.vars - VALGRIND=0 uv run eatmydata pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS} + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Check downgrade (Postgres) + cfg: compile-gcc + test_path: tests/test_downgrade.py + test_db_provider: postgres + setup_old_cln: true + + check-downgrade-liquid: + name: Check downgrade (Liquid) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Check downgrade (Liquid) + cfg: compile-gcc + test_path: tests/test_downgrade.py + test_db_provider: sqlite3 + test_network: liquid-regtest + setup_old_cln: true + + integration-gcc: + name: Test CLN (GCC) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (GCC) + cfg: compile-gcc + test_db_provider: sqlite3 + compiler: gcc + compat: 1 + + integration-clang: + name: Test CLN (Clang) + needs: + - compile-clang + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (Clang) + cfg: compile-clang + test_db_provider: sqlite3 + compiler: clang + compat: 1 + + integration-postgres: + name: Test CLN (Postgres) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (Postgres) + cfg: compile-gcc + test_db_provider: postgres + compiler: gcc + compat: 1 + + integration-liquid: + name: Test CLN (Liquid) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (Liquid) + cfg: compile-gcc + test_db_provider: sqlite3 + compiler: gcc + test_network: liquid-regtest + compat: 1 + + integration-dual-fund: + name: Test CLN (Dual Fund) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (Dual Fund) + cfg: compile-gcc + test_db_provider: sqlite3 + compiler: gcc + experimental_dual_fund: 1 + compat: 1 + + integration-splicing: + name: Test CLN (Splicing) + needs: + - compile-gcc + uses: ./.github/workflows/test-cln.yaml + with: + name: Test CLN (Splicing) + cfg: compile-gcc + test_db_provider: sqlite3 + compiler: gcc + experimental_splicing: 1 + compat: 1 integration-valgrind: - name: Valgrind Test CLN ${{ matrix.name }} - runs-on: ubuntu-22.04 - timeout-minutes: 120 - env: - RUST_PROFILE: release # Has to match the one in the compile step - CFG: compile-gcc - PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --durations=10 + name: Valgrind Test CLN (${{ matrix.shard }}/10) needs: - - compile + - compile-gcc strategy: fail-fast: false matrix: - include: - - NAME: Valgrind (01/10) - PYTEST_OPTS: --test-group=1 --test-group-count=10 - - NAME: Valgrind (02/10) - PYTEST_OPTS: --test-group=2 --test-group-count=10 - - NAME: Valgrind (03/10) - PYTEST_OPTS: --test-group=3 --test-group-count=10 - - NAME: Valgrind (04/10) - PYTEST_OPTS: --test-group=4 --test-group-count=10 - - NAME: Valgrind (05/10) - PYTEST_OPTS: --test-group=5 --test-group-count=10 - - NAME: Valgrind (06/10) - PYTEST_OPTS: --test-group=6 --test-group-count=10 - - NAME: Valgrind (07/10) - PYTEST_OPTS: --test-group=7 --test-group-count=10 - - NAME: Valgrind (08/10) - PYTEST_OPTS: --test-group=8 --test-group-count=10 - - NAME: Valgrind (09/10) - PYTEST_OPTS: --test-group=9 --test-group-count=10 - - NAME: Valgrind (10/10) - PYTEST_OPTS: --test-group=10 --test-group-count=10 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - sudo apt-get update -qq - sudo apt-get install -yyq valgrind - bash -x .github/scripts/setup.sh - - - name: Install bitcoind - run: .github/scripts/install-bitcoind.sh - - - name: Download build - uses: actions/download-artifact@v4 - with: - name: cln-compile-gcc.tar.bz2 - - - name: Unpack build - run: tar -xvjf cln-compile-gcc.tar.bz2 - - - name: Test - env: - SLOW_MACHINE: 1 - TEST_DEBUG: 1 - run: | - VALGRIND=1 uv run eatmydata pytest tests/ -vvv -n 3 ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + uses: ./.github/workflows/test-cln.yaml + with: + name: Valgrind Test CLN + cfg: compile-gcc + pytest_opts: '--test-group-random-seed=42 --timeout=1800 --durations=10' + pytest_workers: 3 + valgrind: 1 + install_valgrind: true + shard_count: 10 + shard_index: ${{ matrix.shard }} + test_db_provider: sqlite3 integration-sanitizers: - name: Sanitizers Test CLN - runs-on: ubuntu-22.04 - timeout-minutes: 120 - env: - RUST_PROFILE: release - SLOW_MACHINE: 1 - TEST_DEBUG: 1 - PYTEST_OPTS: --test-group-random-seed=42 --timeout=1800 --durations=10 + name: Sanitizers Test CLN (${{ matrix.shard }}/10) needs: - - compile + - compile-clang-sanitizers strategy: fail-fast: false matrix: - include: - - NAME: ASan/UBSan (01/10) - PYTEST_OPTS: --test-group=1 --test-group-count=10 - - NAME: ASan/UBSan (02/10) - PYTEST_OPTS: --test-group=2 --test-group-count=10 -n 1 - - NAME: ASan/UBSan (03/10) - PYTEST_OPTS: --test-group=3 --test-group-count=10 - - NAME: ASan/UBSan (04/10) - PYTEST_OPTS: --test-group=4 --test-group-count=10 - - NAME: ASan/UBSan (05/10) - PYTEST_OPTS: --test-group=5 --test-group-count=10 - - NAME: ASan/UBSan (06/10) - PYTEST_OPTS: --test-group=6 --test-group-count=10 - - NAME: ASan/UBSan (07/10) - PYTEST_OPTS: --test-group=7 --test-group-count=10 - - NAME: ASan/UBSan (08/10) - PYTEST_OPTS: --test-group=8 --test-group-count=10 - - NAME: ASan/UBSan (09/10) - PYTEST_OPTS: --test-group=9 --test-group-count=10 - - NAME: ASan/UBSan (10/10) - PYTEST_OPTS: --test-group=10 --test-group-count=10 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - bash -x .github/scripts/setup.sh - - - name: Install bitcoind - run: .github/scripts/install-bitcoind.sh - - - name: Download build - uses: actions/download-artifact@v4 - with: - name: cln-compile-clang-sanitizers.tar.bz2 - - - name: Unpack build - run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2 - - - name: Test - run: | - uv run eatmydata pytest tests/ -vvv -n 2 ${PYTEST_OPTS} ${{ matrix.PYTEST_OPTS }} + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + uses: ./.github/workflows/test-cln.yaml + with: + name: ASan/UBSan Test CLN + cfg: compile-clang-sanitizers + pytest_opts: '--test-group-random-seed=42 --timeout=1800 --durations=10' + pytest_workers: 2 + shard_count: 10 + shard_index: ${{ matrix.shard }} + test_db_provider: sqlite3 update-docs-examples: name: Update examples in doc schemas (disabled temporarily!) @@ -572,7 +379,7 @@ jobs: PYTEST_OPTS: --timeout=1200 --durations=10 TEST_NETWORK: regtest needs: - - compile + - compile-gcc steps: - name: Checkout uses: actions/checkout@v4 @@ -610,7 +417,7 @@ jobs: RUST_PROFILE: release # Has to match the one in the compile step PYTEST_OPTS: --timeout=1200 --durations=10 needs: - - compile + - compile-clang strategy: fail-fast: false matrix: @@ -702,24 +509,37 @@ jobs: name: CI completion runs-on: ubuntu-22.04 needs: - - integration + - integration-gcc + - integration-clang + - integration-postgres + - integration-liquid + - integration-dual-fund + - integration-splicing - check-units - integration-valgrind - integration-sanitizers - min-btc-support - - check-downgrade + - check-downgrade-sqlite + - check-downgrade-postgres + - check-downgrade-liquid if: ${{ always() }} steps: - name: Complete env: - JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC" - INTEGRATION: ${{ needs.integration.result }} + JOB_NAMES: "INTEGRATION_GCC INTEGRATION_CLANG INTEGRATION_POSTGRES INTEGRATION_LIQUID INTEGRATION_DUAL_FUND INTEGRATION_SPLICING CHECK_UNITS VALGRIND SANITIZERS BTC CHECK_DOWNGRADE_SQLITE CHECK_DOWNGRADE_POSTGRES CHECK_DOWNGRADE_LIQUID" + INTEGRATION_GCC: ${{ needs['integration-gcc'].result }} + INTEGRATION_CLANG: ${{ needs['integration-clang'].result }} + INTEGRATION_POSTGRES: ${{ needs['integration-postgres'].result }} + INTEGRATION_LIQUID: ${{ needs['integration-liquid'].result }} + INTEGRATION_DUAL_FUND: ${{ needs['integration-dual-fund'].result }} + INTEGRATION_SPLICING: ${{ needs['integration-splicing'].result }} CHECK_UNITS: ${{ needs['check-units'].result }} VALGRIND: ${{ needs['integration-valgrind'].result }} SANITIZERS: ${{ needs['integration-sanitizers'].result }} - DOCS: ${{ needs['update-docs-examples'].result }} BTC: ${{ needs['min-btc-support'].result }} - CHECK_DOWNGRADE: ${{ needs['check-downgrade'].result }} + CHECK_DOWNGRADE_SQLITE: ${{ needs['check-downgrade-sqlite'].result }} + CHECK_DOWNGRADE_POSTGRES: ${{ needs['check-downgrade-postgres'].result }} + CHECK_DOWNGRADE_LIQUID: ${{ needs['check-downgrade-liquid'].result }} run: | failed="" for name in $JOB_NAMES; do diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index 71ca8e9d235f..ea41d2ea46ad 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -2,11 +2,22 @@ name: Mac OS pytest on: pull_request: + workflow_dispatch: jobs: + build-macos: + name: Build CLN (macOS) + uses: ./.github/workflows/build-cln.yaml + with: + cfg: macos-clang + compiler: clang + os: macos-14 + smoke-test: name: Smoke Test macOS runs-on: macos-14 timeout-minutes: 120 + needs: + - build-macos strategy: fail-fast: true matrix: @@ -25,32 +36,13 @@ jobs: sudo mv bitcoin-${BITCOIND_VERSION}/bin/* /usr/local/bin rm -rf bitcoin-${BITCOIND_VERSION}-${TARGET_ARCH}.tar.gz bitcoin-${BITCOIND_VERSION} - - name: Set up Python 3.10 - uses: actions/setup-python@v5 + - name: Download build + uses: actions/download-artifact@v4 with: - python-version: "3.10" - - - name: Install uv - uses: astral-sh/setup-uv@v5 - - - name: Install dependencies - run: | - export PATH="/usr/local/opt:/Users/runner/.local/bin:/opt/homebrew/bin/python3.10/bin:$PATH" - - brew install gnu-sed autoconf automake libtool protobuf openssl lowdown libsodium - - # https://github.com/grpc/grpc/issues/31737#issuecomment-1323796842 - export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 - export GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 - uv sync --all-groups - - - name: Build and install CLN - run: | - export CPATH=/opt/homebrew/include - export LIBRARY_PATH=/opt/homebrew/lib + name: cln-macos-clang.tar.bz2 - uv run ./configure --disable-valgrind --disable-compat - uv run make + - name: Unpack CLN + run: tar -xaf cln-macos-clang.tar.bz2 - name: Start bitcoind in regtest mode run: | diff --git a/.github/workflows/test-cln.yaml b/.github/workflows/test-cln.yaml new file mode 100644 index 000000000000..5cdf949b65bc --- /dev/null +++ b/.github/workflows/test-cln.yaml @@ -0,0 +1,198 @@ +--- +name: Test CLN +on: + workflow_call: + inputs: + name: + description: 'Test name for display' + required: true + type: string + cfg: + description: 'Build configuration to use (e.g., compile-gcc)' + required: true + type: string + test_path: + description: 'Path/pattern for pytest' + required: false + type: string + default: 'tests/' + pytest_opts: + description: 'Base pytest options' + required: false + type: string + default: '--timeout=1200 --durations=10' + matrix_pytest_opts: + description: 'Additional pytest options from matrix (e.g., test groups)' + required: false + type: string + default: '' + pytest_workers: + description: 'Number of pytest parallel workers (-n flag)' + required: false + type: number + default: 10 + valgrind: + description: 'Enable valgrind (0 or 1)' + required: false + type: number + default: 0 + test_debug: + description: 'Enable test debug (0 or 1)' + required: false + type: number + default: 1 + test_db_provider: + description: 'Database provider (sqlite3, postgres, or empty)' + required: false + type: string + default: '' + test_network: + description: 'Test network (regtest, liquid-regtest)' + required: false + type: string + default: 'regtest' + experimental_dual_fund: + description: 'Enable experimental dual fund' + required: false + type: number + default: 0 + experimental_splicing: + description: 'Enable experimental splicing' + required: false + type: number + default: 0 + lightningd_postgres_no_vacuum: + description: 'Disable postgres vacuum' + required: false + type: number + default: 1 + slow_machine: + description: 'Enable slow machine mode' + required: false + type: number + default: 1 + compat: + description: 'Enable compatibility mode' + required: false + type: number + default: 0 + compiler: + description: 'Compiler used (for env var)' + required: false + type: string + default: '' + rust_profile: + description: 'Rust profile' + required: false + type: string + default: 'release' + install_valgrind: + description: 'Install valgrind package before tests' + required: false + type: boolean + default: false + install_bitcoind: + description: 'Install bitcoind (true) or skip (false for custom Bitcoin install)' + required: false + type: boolean + default: true + setup_old_cln: + description: 'Download and setup old CLN for downgrade tests' + required: false + type: boolean + default: false + old_cln_version: + description: 'Version of old CLN to download (e.g., v25.09)' + required: false + type: string + default: 'v25.09' + shard_count: + description: 'Number of test shards (1 = no sharding, >1 = enable sharding)' + required: false + type: number + default: 1 + shard_index: + description: 'Shard index for this run (1-based, used when shard_count > 1)' + required: false + type: number + default: 1 + +jobs: + test: + name: ${{ inputs.shard_count > 1 && format('{0} ({1}/{2})', inputs.name, inputs.shard_index, inputs.shard_count) || inputs.name }} + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + RUST_PROFILE: ${{ inputs.rust_profile }} + PYTEST_OPTS: ${{ inputs.pytest_opts }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Install dependencies + run: | + bash -x .github/scripts/setup.sh + + - name: Install valgrind + if: ${{ inputs.install_valgrind }} + run: | + sudo apt-get update -qq + sudo apt-get install -yyq valgrind + + - name: Install bitcoind + if: ${{ inputs.install_bitcoind }} + env: + TEST_NETWORK: ${{ inputs.test_network }} + run: .github/scripts/install-bitcoind.sh + + - name: Download build + uses: actions/download-artifact@v4 + with: + name: cln-${{ inputs.cfg }}.tar.bz2 + + - name: Unpack pre-built CLN + run: tar -xaf cln-${{ inputs.cfg }}.tar.bz2 + + - name: Fetch and unpack previous CLN + if: ${{ inputs.setup_old_cln }} + run: | + mkdir /tmp/old-cln + cd /tmp/old-cln + wget https://github.com/ElementsProject/lightning/releases/download/${{ inputs.old_cln_version }}/clightning-${{ inputs.old_cln_version }}-Ubuntu-22.04-amd64.tar.xz + tar -xaf clightning-${{ inputs.old_cln_version }}-Ubuntu-22.04-amd64.tar.xz + + - name: Switch network + if: ${{ inputs.test_network == 'liquid-regtest' }} + run: | + # Loading the network from config.vars rather than the envvar is a terrible idea... + sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars + cat config.vars + + - name: Test + env: + COMPILER: ${{ inputs.compiler }} + EXPERIMENTAL_DUAL_FUND: ${{ inputs.experimental_dual_fund }} + EXPERIMENTAL_SPLICING: ${{ inputs.experimental_splicing }} + COMPAT: ${{ inputs.compat }} + CFG: ${{ inputs.cfg }} + SLOW_MACHINE: ${{ inputs.slow_machine }} + PYTEST_PAR: ${{ inputs.pytest_workers }} + TEST_DEBUG: ${{ inputs.test_debug }} + TEST_DB_PROVIDER: ${{ inputs.test_db_provider }} + TEST_NETWORK: ${{ inputs.test_network }} + LIGHTNINGD_POSTGRES_NO_VACUUM: ${{ inputs.lightningd_postgres_no_vacuum }} + VALGRIND: ${{ inputs.valgrind }} + PREV_LIGHTNINGD: ${{ inputs.setup_old_cln && '/tmp/old-cln/usr/bin/lightningd' || '' }} + SHARD_OPTS: ${{ inputs.shard_count > 1 && format('--test-group={0} --test-group-count={1}', inputs.shard_index, inputs.shard_count) || '' }} + run: | + env + cat config.vars + uv run eatmydata pytest ${{ inputs.test_path }} -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS} ${SHARD_OPTS} ${{ inputs.matrix_pytest_opts }}