Backports and changes to fix CI

.cirrus.yml

@@ -66,20 +66,6 @@ compute_credits_template: &CREDITS_TEMPLATE
   # Only use credits for pull requests to the main repo
   use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'ElementsProject/elements' && $CIRRUS_PR != ""
 
-task:
-  name: 'lint [bionic]'
-  << : *BASE_TEMPLATE
-  container:
-    image: ubuntu:bionic  # For python 3.6, oldest supported version according to doc/dependencies.md
-    cpu: 1
-    memory: 1G
-  # For faster CI feedback, immediately schedule the linters
-  << : *CREDITS_TEMPLATE
-  lint_script:
-    - ./ci/lint_run_all.sh
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-
 task:
   name: "Win64 native [msvc]"
   << : *FILTER_TEMPLATE
@@ -160,139 +146,26 @@ task:
     - python build_msvc\msvc-autogen.py
     - msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo
   unit_tests_script:
-    - src\test_elements.exe -l test_suite
-    - src\bench_elements.exe > NUL
-    - python test\util\test_runner.py
+    # - src\test_elements.exe -l test_suite
+    # - src\bench_elements.exe > NUL
+    # - python test\util\test_runner.py
     - python test\util\rpcauth-test.py
   functional_tests_script:
     # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted".
     # See: https://docs.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance
     - netsh int ipv4 set dynamicport tcp start=1025 num=64511
     - netsh int ipv6 set dynamicport tcp start=1025 num=64511
+    - mkdir artifactsdir
     # Exclude feature_dbcrash for now due to timeout
     # Exclude also wallet_avoidreuse due to timeout
     # Ignore failures for now, need to investigate but we really don't use native win64 builds
-    - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash,wallet_avoidreuse || true
-
-task:
-  name: 'ARM [unit tests, no functional tests] [bullseye]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: debian:bullseye
-    docker_arguments:
-      CI_IMAGE_NAME_TAG: debian:bullseye
-  << : *CREDITS_TEMPLATE
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_arm.sh"
-
-task:
-  name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:jammy
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_win64.sh"
-
-task:
-  name: '32-bit + dash [gui] [Rocky 8]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: quay.io/rockylinux/rockylinux:8
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    PACKAGE_MANAGER_INSTALL: "yum install -y"
-    FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
+    - python test\functional\test_runner.py --tmpdirprefix=artifactsdir/ --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash,wallet_avoidreuse,feature_trim_headers || true
 
-task:
-  name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [bionic]'
-  previous_releases_cache:
-    folder: "releases"
-  << : *GLOBAL_TASK_TEMPLATE
-  << : *PERSISTENT_WORKER_TEMPLATE
-  env:
-    << : *PERSISTENT_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"
-
-task:
-  name: '[TSan, depends, gui] [2404]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:24.04
-    cpu: 6  # Increase CPU and Memory to avoid timeout
-    memory: 24G
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
-    MAKEJOBS: "-j2"  # Avoid excessive memory use due to MSan
-
-task:
-  name: '[MSan, depends] [focal]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:focal
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
-
-task:
-  name: '[ASan + LSan + UBSan + integer, no depends] [jammy]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:jammy
-    memory: 16G # ELEMENTS: need more memory
-    cpu: 4  # ELEMENTS: cirrus wants more CPUs if you want more memory
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
-
-task:
-  name: '[fuzzer,address,undefined,integer, no depends] [jammy]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:jammy
-    cpu: 8  # Increase CPU and memory to avoid timeout
-    memory: 16G
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
-
-task:
-  name: '[multiprocess, i686, DEBUG] [focal]'
-  # Disable for Elements for now; Multiprocess build is not supported or tested and fails CI.
-  only_if: false
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:focal
-    cpu: 4
-    memory: 16G  # The default memory is sometimes just a bit too small, so double everything
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
-
-task:
-  name: '[no wallet, libbitcoinkernel] [bionic]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:bionic
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh"
-
-task:
-  name: 'macOS 10.15 [gui, no tests] [focal]'
-  << : *BASE_TEMPLATE
-  macos_sdk_cache:
-    folder: "depends/SDKs/$MACOS_SDK"
-    fingerprint_key: "$MACOS_SDK"
-  << : *MAIN_TEMPLATE
-  container:
-    image: ubuntu:focal
-  env:
-    MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers"
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_mac.sh"
+  artifacts:
+    paths:
+      - artifactsdir/**/*.log
+      - artifactsdir/**/tmp*
+      - artifactsdir/**/*.conf
 
 task:
   name: 'macOS 13 native arm64 [gui, sqlite only] [no depends]'
@@ -305,20 +178,3 @@ task:
     CI_USE_APT_INSTALL: "no"
     PACKAGE_MANAGER_INSTALL: "echo"  # Nothing to do
     FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh"
-
-task:
-  name: 'ARM64 Android APK [focal]'
-  # Disable for Elements; Android build is broken and unsupported.
-  only_if: false
-  android_sdk_cache:
-    folder: "depends/SDKs/android"
-    fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.1.7779620"
-  depends_sources_cache:
-    folder: "depends/sources"
-    fingerprint_script: git rev-list -1 HEAD ./depends
-  << : *MAIN_TEMPLATE
-  container:
-    image: ubuntu:focal
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_android.sh"

ci/lint/06_script.sh

@@ -19,7 +19,6 @@ test/lint/git-subtree-check.sh src/crypto/ctaes
 test/lint/git-subtree-check.sh src/secp256k1
 test/lint/git-subtree-check.sh src/simplicity
 test/lint/git-subtree-check.sh src/minisketch
-test/lint/git-subtree-check.sh src/univalue
 test/lint/git-subtree-check.sh src/leveldb
 test/lint/git-subtree-check.sh src/crc32c
 test/lint/check-doc.py

ci/test/04_install.sh

@@ -11,7 +11,7 @@ if [[ $QEMU_USER_CMD == qemu-s390* ]]; then
 fi
 
 if [ "$CI_OS_NAME" == "macos" ]; then
-  sudo -H pip3 install --upgrade --break-system-packages pip
+  sudo -H pip3 install --upgrade pip
   # shellcheck disable=SC2086
   IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
 fi

configure.ac

@@ -445,6 +445,7 @@ if test "$CXXFLAGS_overridden" = "no"; then
   AX_CHECK_COMPILE_FLAG([-Wthread-safety], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"], [], [$CXXFLAG_WERROR])
   AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"], [], [$CXXFLAG_WERROR])
   AX_CHECK_COMPILE_FLAG([-Wredundant-decls], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"], [], [$CXXFLAG_WERROR])
+  AX_CHECK_COMPILE_FLAG([-Wunused-result],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-result"],,[[$CXXFLAG_WERROR]])
   AX_CHECK_COMPILE_FLAG([-Wunused-member-function], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"], [], [$CXXFLAG_WERROR])
   AX_CHECK_COMPILE_FLAG([-Wdate-time], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"], [], [$CXXFLAG_WERROR])
   AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"], [], [$CXXFLAG_WERROR])

src/addrdb.cpp

@@ -33,10 +33,9 @@ bool SerializeDB(Stream& stream, const Data& data)
 {
     // Write and commit header, data
     try {
-        CHashWriter hasher(stream.GetType(), stream.GetVersion());
-        stream << Params().MessageStart() << data;
-        hasher << Params().MessageStart() << data;
-        stream << hasher.GetHash();
+        HashedSourceWriter hashwriter{stream};
+        hashwriter << Params().MessageStart() << data;
+        stream << hashwriter.GetHash();
     } catch (const std::exception& e) {
         return error("%s: Serialize or I/O error - %s", __func__, e.what());
     }

src/addrman.cpp

@@ -1171,8 +1171,7 @@ void AddrMan::Unserialize(Stream& s_)
 }
 
 // explicit instantiation
-template void AddrMan::Serialize(CHashWriter& s) const;
-template void AddrMan::Serialize(CAutoFile& s) const;
+template void AddrMan::Serialize(HashedSourceWriter<CAutoFile>& s) const;
 template void AddrMan::Serialize(CDataStream& s) const;
 template void AddrMan::Unserialize(CAutoFile& s);
 template void AddrMan::Unserialize(CHashVerifier<CAutoFile>& s);

src/bitcoin-cli.cpp

@@ -848,7 +848,7 @@ static UniValue ConnectAndCallRPC(BaseRequestHandler* rh, const std::string& str
         try {
             response = CallRPC(rh, strMethod, args, rpcwallet);
             if (fWait) {
-                const UniValue& error = find_value(response, "error");
+                const UniValue& error = response.find_value("error");
                 if (!error.isNull() && error["code"].get_int() == RPC_IN_WARMUP) {
                     throw CConnectionFailed("server in warmup");
                 }
@@ -877,8 +877,8 @@ static void ParseResult(const UniValue& result, std::string& strPrint)
 static void ParseError(const UniValue& error, std::string& strPrint, int& nRet)
 {
     if (error.isObject()) {
-        const UniValue& err_code = find_value(error, "code");
-        const UniValue& err_msg = find_value(error, "message");
+        const UniValue& err_code = error.find_value("code");
+        const UniValue& err_msg = error.find_value("message");
         if (!err_code.isNull()) {
             strPrint = "error code: " + err_code.getValStr() + "\n";
         }
@@ -904,15 +904,15 @@ static void GetWalletBalances(UniValue& result)
 {
     DefaultRequestHandler rh;
     const UniValue listwallets = ConnectAndCallRPC(&rh, "listwallets", /* args=*/{});
-    if (!find_value(listwallets, "error").isNull()) return;
-    const UniValue& wallets = find_value(listwallets, "result");
+    if (!listwallets.find_value("error").isNull()) return;
+    const UniValue& wallets = listwallets.find_value("result");
     if (wallets.size() <= 1) return;
 
     UniValue balances(UniValue::VOBJ);
     for (const UniValue& wallet : wallets.getValues()) {
         const std::string wallet_name = wallet.get_str();
         const UniValue getbalances = ConnectAndCallRPC(&rh, "getbalances", /* args=*/{}, wallet_name);
-        const UniValue& balance = find_value(getbalances, "result")["mine"]["trusted"];
+        const UniValue& balance = getbalances.find_value("result")["mine"]["trusted"];
         balances.pushKV(wallet_name, balance);
     }
     result.pushKV("balances", balances);
@@ -948,7 +948,7 @@ static void GetProgressBar(double progress, std::string& progress_bar)
  */
 static void ParseGetInfoResult(UniValue& result)
 {
-    if (!find_value(result, "error").isNull()) return;
+    if (!result.find_value("error").isNull()) return;
 
     std::string RESET, GREEN, BLUE, YELLOW, MAGENTA, CYAN;
     bool should_colorize = false;
@@ -1157,9 +1157,9 @@ static int CommandLineRPC(int argc, char *argv[])
             rh.reset(new NetinfoRequestHandler());
         } else if (gArgs.GetBoolArg("-generate", false)) {
             const UniValue getnewaddress{GetNewAddress()};
-            const UniValue& error{find_value(getnewaddress, "error")};
+            const UniValue& error{getnewaddress.find_value("error")};
             if (error.isNull()) {
-                SetGenerateToAddressArgs(find_value(getnewaddress, "result").get_str(), args);
+                SetGenerateToAddressArgs(getnewaddress.find_value("result").get_str(), args);
                 rh.reset(new GenerateToAddressRequestHandler());
             } else {
                 ParseError(error, strPrint, nRet);
@@ -1181,8 +1181,8 @@ static int CommandLineRPC(int argc, char *argv[])
             const UniValue reply = ConnectAndCallRPC(rh.get(), method, args, wallet_name);
 
             // Parse reply
-            UniValue result = find_value(reply, "result");
-            const UniValue& error = find_value(reply, "error");
+            UniValue result = reply.find_value("result");
+            const UniValue& error = reply.find_value("error");
             if (error.isNull()) {
                 if (gArgs.GetBoolArg("-getinfo", false)) {
                     if (!gArgs.IsArgSet("-rpcwallet")) {

src/external_signer.cpp

@@ -30,15 +30,15 @@ bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalS
     }
     for (UniValue signer : result.getValues()) {
         // Check for error
-        const UniValue& error = find_value(signer, "error");
+        const UniValue& error = signer.find_value("error");
         if (!error.isNull()) {
             if (!error.isStr()) {
                 throw std::runtime_error(strprintf("'%s' error", command));
             }
             throw std::runtime_error(strprintf("'%s' error: %s", command, error.getValStr()));
         }
         // Check if fingerprint is present
-        const UniValue& fingerprint = find_value(signer, "fingerprint");
+        const UniValue& fingerprint = signer.find_value("fingerprint");
         if (fingerprint.isNull()) {
             throw std::runtime_error(strprintf("'%s' received invalid response, missing signer fingerprint", command));
         }
@@ -50,7 +50,7 @@ bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalS
         }
         if (duplicate) break;
         std::string name = "";
-        const UniValue& model_field = find_value(signer, "model");
+        const UniValue& model_field = signer.find_value("model");
         if (model_field.isStr() && model_field.getValStr() != "") {
             name += model_field.getValStr();
         }
@@ -93,19 +93,19 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str
 
     const UniValue signer_result = RunCommandParseJSON(command, stdinStr);
 
-    if (find_value(signer_result, "error").isStr()) {
-        error = find_value(signer_result, "error").get_str();
+    if (signer_result.find_value("error").isStr()) {
+        error = signer_result.find_value("error").get_str();
         return false;
     }
 
-    if (!find_value(signer_result, "psbt").isStr()) {
+    if (!signer_result.find_value("psbt").isStr()) {
         error = "Unexpected result from signer";
         return false;
     }
 
     PartiallySignedTransaction signer_psbtx;
     std::string signer_psbt_error;
-    if (!DecodeBase64PSBT(signer_psbtx, find_value(signer_result, "psbt").get_str(), signer_psbt_error)) {
+    if (!DecodeBase64PSBT(signer_psbtx, signer_result.find_value("psbt").get_str(), signer_psbt_error)) {
         error = strprintf("TX decode failed %s", signer_psbt_error);
         return false;
     }

src/hash.h

@@ -188,6 +188,30 @@ class CHashVerifier : public CHashWriter
     }
 };
 
+/** Writes data to an underlying source stream, while hashing the written data. */
+template <typename Source>
+class HashedSourceWriter : public CHashWriter
+{
+private:
+    Source& m_source;
+
+public:
+    explicit HashedSourceWriter(Source& source LIFETIMEBOUND) : CHashWriter{source.GetType(), source.GetVersion()}, m_source{source} {}
+
+    void write(Span<const std::byte> src)
+    {
+        m_source.write(src);
+        CHashWriter::write(src);
+    }
+
+    template <typename T>
+    HashedSourceWriter& operator<<(const T& obj)
+    {
+        ::Serialize(*this, obj);
+        return *this;
+    }
+};
+
 /** Compute the 256-bit hash of an object's serialization, with optional sighash byte. */
 template<typename T>
 uint256 SerializeHash(const T& obj, int nType=SER_GETHASH, int nVersion=PROTOCOL_VERSION)