diff --git a/.cirrus.yml b/.cirrus.yml index 240e2cf70536..bd192dfea3fa 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -30,7 +30,7 @@ base_template: &BASE_TEMPLATE - git config --global user.name "ci" - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH - - git merge FETCH_HEAD # Merge base to detect silent merge conflicts +# - git merge FETCH_HEAD # Merge base to detect silent merge conflicts main_template: &MAIN_TEMPLATE timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out @@ -57,183 +57,183 @@ compute_credits_template: &CREDITS_TEMPLATE # Only use credits for pull requests to the main repo use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != "" -task: - name: 'lint [bionic]' - << : *BASE_TEMPLATE - container: - image: ubuntu:bionic # For python 3.6, oldest supported version according to doc/dependencies.md - cpu: 1 - memory: 1G - # For faster CI feedback, immediately schedule the linters - << : *CREDITS_TEMPLATE - lint_script: - - ./ci/lint_run_all.sh - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - -task: - name: "Win64 native [msvc]" - << : *FILTER_TEMPLATE - windows_container: - cpu: 4 - memory: 8G - image: cirrusci/windowsservercore:visualstudio2019 - timeout_in: 120m - env: - PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' - PYTHONUTF8: 1 - CI_VCPKG_TAG: '2021.05.12' - VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' - VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' - QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip' - QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.15.2.zip' - QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.2' - QTBASEDIR: 'C:\Qt_static' - x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' - IgnoreWarnIntDirInTempDetected: 'true' - merge_script: - - git config --global user.email "ci@ci.ci" - - git config --global user.name "ci" - # Windows filesystem loses the executable bit, and all of the executable - # files are considered "modified" now. It will break the following `git merge` - # command. The next two commands make git ignore this issue. - - git config core.filemode false - - git reset --hard - - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL $env:CIRRUS_BASE_BRANCH; git merge FETCH_HEAD; } - msvc_qt_built_cache: - folder: "%QTBASEDIR%" - reupload_on_changes: false - fingerprint_script: - - echo %QT_DOWNLOAD_URL% - - msbuild -version - populate_script: - - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip - - mkdir C:\jom - - tar -xf C:\jom.zip -C C:\jom - - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL% - - tar -xf %QT_LOCAL_PATH% -C C:\ - - '%x64_NATIVE_TOOLS%' - - cd %QT_SOURCE_DIR% - - mkdir build - - cd build - - ..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix %QTBASEDIR% - - jom - - jom install - vcpkg_tools_cache: - folder: '%VCPKG_DOWNLOADS%\tools' - reupload_on_changes: false - fingerprint_script: - - echo %CI_VCPKG_TAG% - - msbuild -version - vcpkg_binary_cache: - folder: '%VCPKG_DEFAULT_BINARY_CACHE%' - reupload_on_changes: true - fingerprint_script: - - echo %CI_VCPKG_TAG% - - msbuild -version - populate_script: - - mkdir %VCPKG_DEFAULT_BINARY_CACHE% - install_python_script: - - choco install --yes --no-progress python3 --version=3.9.6 - - pip install zmq - - python -VV - install_vcpkg_script: - - cd .. - - git clone --quiet https://github.com/microsoft/vcpkg.git - - cd vcpkg - - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG% - - .\bootstrap-vcpkg -disableMetrics - - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake - - .\vcpkg integrate install - - .\vcpkg version - build_script: - - cd %CIRRUS_WORKING_DIR% - - python build_msvc\msvc-autogen.py - - msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo - unit_tests_script: - - src\test_bitcoin.exe -l test_suite - - src\bench_bitcoin.exe > NUL - - python test\util\test_runner.py - - python test\util\rpcauth-test.py - functional_tests_script: - # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted". - # See: https://docs.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance - - netsh int ipv4 set dynamicport tcp start=1025 num=64511 - - netsh int ipv6 set dynamicport tcp start=1025 num=64511 - # Exclude feature_dbcrash for now due to timeout - - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash +#task: +# name: 'lint [bionic]' +# << : *BASE_TEMPLATE +# container: +# image: ubuntu:bionic # For python 3.6, oldest supported version according to doc/dependencies.md +# cpu: 1 +# memory: 1G +# # For faster CI feedback, immediately schedule the linters +# << : *CREDITS_TEMPLATE +# lint_script: +# - ./ci/lint_run_all.sh +# env: +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV -task: - name: 'ARM [unit tests, no functional tests] [bullseye]' - << : *GLOBAL_TASK_TEMPLATE - arm_container: - image: debian:bullseye - cpu: 2 - memory: 8G - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_arm.sh" - QEMU_USER_CMD: "" # Disable qemu and run the test natively - -task: - name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [focal]' - << : *GLOBAL_TASK_TEMPLATE - container: - image: ubuntu:focal - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_win64.sh" - -task: - name: '32-bit + dash [gui] [CentOS 8]' - << : *GLOBAL_TASK_TEMPLATE - container: - image: centos:8 - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - PACKAGE_MANAGER_INSTALL: "yum install -y" - FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" - -task: - name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [bionic]' - previous_releases_cache: - folder: "releases" - << : *GLOBAL_TASK_TEMPLATE - << : *PERSISTENT_WORKER_TEMPLATE - env: - << : *PERSISTENT_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh" - -task: - name: '[TSan, depends, gui] [jammy]' - << : *GLOBAL_TASK_TEMPLATE - container: - image: ubuntu:jammy - cpu: 6 # Increase CPU and Memory to avoid timeout - memory: 24G - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" - -task: - name: '[MSan, depends] [focal]' - << : *GLOBAL_TASK_TEMPLATE - container: - image: ubuntu:focal - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" - MAKEJOBS: "-j4" # Avoid excessive memory use due to MSan - -task: - name: '[ASan + LSan + UBSan + integer, no depends] [jammy]' - << : *GLOBAL_TASK_TEMPLATE - container: - image: ubuntu:jammy - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" - MAKEJOBS: "-j4" # Avoid excessive memory use +#task: +# name: "Win64 native [msvc]" +# << : *FILTER_TEMPLATE +# windows_container: +# cpu: 4 +# memory: 8G +# image: cirrusci/windowsservercore:visualstudio2019 +# timeout_in: 120m +# env: +# PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' +# PYTHONUTF8: 1 +# CI_VCPKG_TAG: '2021.05.12' +# VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' +# VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' +# QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip' +# QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.15.2.zip' +# QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.2' +# QTBASEDIR: 'C:\Qt_static' +# x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' +# IgnoreWarnIntDirInTempDetected: 'true' +# merge_script: +# - git config --global user.email "ci@ci.ci" +# - git config --global user.name "ci" +# # Windows filesystem loses the executable bit, and all of the executable +# # files are considered "modified" now. It will break the following `git merge` +# # command. The next two commands make git ignore this issue. +# - git config core.filemode false +# - git reset --hard +# - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL $env:CIRRUS_BASE_BRANCH; git merge FETCH_HEAD; } +# msvc_qt_built_cache: +# folder: "%QTBASEDIR%" +# reupload_on_changes: false +# fingerprint_script: +# - echo %QT_DOWNLOAD_URL% +# - msbuild -version +# populate_script: +# - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip +# - mkdir C:\jom +# - tar -xf C:\jom.zip -C C:\jom +# - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL% +# - tar -xf %QT_LOCAL_PATH% -C C:\ +# - '%x64_NATIVE_TOOLS%' +# - cd %QT_SOURCE_DIR% +# - mkdir build +# - cd build +# - ..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix %QTBASEDIR% +# - jom +# - jom install +# vcpkg_tools_cache: +# folder: '%VCPKG_DOWNLOADS%\tools' +# reupload_on_changes: false +# fingerprint_script: +# - echo %CI_VCPKG_TAG% +# - msbuild -version +# vcpkg_binary_cache: +# folder: '%VCPKG_DEFAULT_BINARY_CACHE%' +# reupload_on_changes: true +# fingerprint_script: +# - echo %CI_VCPKG_TAG% +# - msbuild -version +# populate_script: +# - mkdir %VCPKG_DEFAULT_BINARY_CACHE% +# install_python_script: +# - choco install --yes --no-progress python3 --version=3.9.6 +# - pip install zmq +# - python -VV +# install_vcpkg_script: +# - cd .. +# - git clone --quiet https://github.com/microsoft/vcpkg.git +# - cd vcpkg +# - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG% +# - .\bootstrap-vcpkg -disableMetrics +# - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake +# - .\vcpkg integrate install +# - .\vcpkg version +# build_script: +# - cd %CIRRUS_WORKING_DIR% +# - python build_msvc\msvc-autogen.py +# - msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo +# unit_tests_script: +# - src\test_bitcoin.exe -l test_suite +# - src\bench_bitcoin.exe > NUL +# - python test\util\test_runner.py +# - python test\util\rpcauth-test.py +# functional_tests_script: +# # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted". +# # See: https://docs.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance +# - netsh int ipv4 set dynamicport tcp start=1025 num=64511 +# - netsh int ipv6 set dynamicport tcp start=1025 num=64511 +# # Exclude feature_dbcrash for now due to timeout +# - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash +# +#task: +# name: 'ARM [unit tests, no functional tests] [bullseye]' +# << : *GLOBAL_TASK_TEMPLATE +# arm_container: +# image: debian:bullseye +# cpu: 2 +# memory: 8G +# env: +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV +# FILE_ENV: "./ci/test/00_setup_env_arm.sh" +# QEMU_USER_CMD: "" # Disable qemu and run the test natively +# +#task: +# name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [focal]' +# << : *GLOBAL_TASK_TEMPLATE +# container: +# image: ubuntu:focal +# env: +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV +# FILE_ENV: "./ci/test/00_setup_env_win64.sh" +# +#task: +# name: '32-bit + dash [gui] [CentOS 8]' +# << : *GLOBAL_TASK_TEMPLATE +# container: +# image: centos:8 +# env: +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV +# PACKAGE_MANAGER_INSTALL: "yum install -y" +# FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" +# +#task: +# name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [bionic]' +# previous_releases_cache: +# folder: "releases" +# << : *GLOBAL_TASK_TEMPLATE +# << : *PERSISTENT_WORKER_TEMPLATE +# env: +# << : *PERSISTENT_WORKER_TEMPLATE_ENV +# FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh" +# +#task: +# name: '[TSan, depends, no gui] [jammy]' +# << : *GLOBAL_TASK_TEMPLATE +# container: +# image: ubuntu:jammy +# cpu: 6 # Increase CPU and Memory to avoid timeout +# memory: 24G +# env: +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV +# FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" +# +#task: +# name: '[MSan, depends] [focal]' +# << : *GLOBAL_TASK_TEMPLATE +# container: +# image: ubuntu:focal +# env: +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV +# FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" +# MAKEJOBS: "-j4" # Avoid excessive memory use due to MSan +# +#task: +# name: '[ASan + LSan + UBSan + integer, no depends] [jammy]' +# << : *GLOBAL_TASK_TEMPLATE +# container: +# image: ubuntu:jammy +# env: +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV +# FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" +# MAKEJOBS: "-j4" # Avoid excessive memory use task: name: '[fuzzer,address,undefined,integer, no depends] [focal]' @@ -246,67 +246,70 @@ task: env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" +# python_script: +## - if [ "$CIRRUS_PR" = "" ]; then python src/test/fuzz/script.py; fi +# - python src/test/fuzz/script.py -task: - name: '[multiprocess, i686, DEBUG] [focal]' - << : *GLOBAL_TASK_TEMPLATE - container: - image: ubuntu:focal - cpu: 4 - memory: 16G # The default memory is sometimes just a bit too small, so double everything - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" - -task: - name: '[no wallet] [bionic]' - << : *GLOBAL_TASK_TEMPLATE - container: - image: ubuntu:bionic - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh" - -task: - name: 'macOS 10.15 [gui, no tests] [focal]' - << : *BASE_TEMPLATE - macos_sdk_cache: - folder: "depends/SDKs/$MACOS_SDK" - fingerprint_key: "$MACOS_SDK" - << : *MAIN_TEMPLATE - container: - image: ubuntu:focal - env: - MACOS_SDK: "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers" - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_mac.sh" - -task: - name: 'macOS 12 native [gui, system sqlite only] [no depends]' - brew_install_script: - - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt - << : *GLOBAL_TASK_TEMPLATE - osx_instance: - # Use latest image, but hardcode version to avoid silent upgrades (and breaks) - image: monterey-xcode-13.2 # https://cirrus-ci.org/guide/macOS - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - CI_USE_APT_INSTALL: "no" - PACKAGE_MANAGER_INSTALL: "echo" # Nothing to do - FILE_ENV: "./ci/test/00_setup_env_mac_host.sh" - -task: - name: 'ARM64 Android APK [focal]' - << : *BASE_TEMPLATE - android_sdk_cache: - folder: "depends/SDKs/android" - fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.1.7779620" - depends_sources_cache: - folder: "depends/sources" - fingerprint_script: git rev-list -1 HEAD ./depends - << : *MAIN_TEMPLATE - container: - image: ubuntu:focal - env: - << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_android.sh" +#task: +# name: '[multiprocess, i686, DEBUG] [focal]' +# << : *GLOBAL_TASK_TEMPLATE +# container: +# image: ubuntu:focal +# cpu: 4 +# memory: 16G # The default memory is sometimes just a bit too small, so double everything +# env: +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV +# FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" +# +#task: +# name: '[no wallet] [bionic]' +# << : *GLOBAL_TASK_TEMPLATE +# container: +# image: ubuntu:bionic +# env: +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV +# FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh" +# +#task: +# name: 'macOS 10.15 [gui, no tests] [focal]' +# << : *BASE_TEMPLATE +# macos_sdk_cache: +# folder: "depends/SDKs/$MACOS_SDK" +# fingerprint_key: "$MACOS_SDK" +# << : *MAIN_TEMPLATE +# container: +# image: ubuntu:focal +# env: +# MACOS_SDK: "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers" +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV +# FILE_ENV: "./ci/test/00_setup_env_mac.sh" +# +#task: +# name: 'macOS 12 native [gui, system sqlite only] [no depends]' +# brew_install_script: +# - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt +# << : *GLOBAL_TASK_TEMPLATE +# osx_instance: +# # Use latest image, but hardcode version to avoid silent upgrades (and breaks) +# image: monterey-xcode-13.2 # https://cirrus-ci.org/guide/macOS +# env: +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV +# CI_USE_APT_INSTALL: "no" +# PACKAGE_MANAGER_INSTALL: "echo" # Nothing to do +# FILE_ENV: "./ci/test/00_setup_env_mac_host.sh" +# +#task: +# name: 'ARM64 Android APK [focal]' +# << : *BASE_TEMPLATE +# android_sdk_cache: +# folder: "depends/SDKs/android" +# fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.1.7779620" +# depends_sources_cache: +# folder: "depends/sources" +# fingerprint_script: git rev-list -1 HEAD ./depends +# << : *MAIN_TEMPLATE +# container: +# image: ubuntu:focal +# env: +# << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV +# FILE_ENV: "./ci/test/00_setup_env_android.sh" diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 801745d0c6b6..12e0f748cf55 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -240,6 +240,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/crypto_chacha20_poly1305_aead.cpp \ test/fuzz/crypto_common.cpp \ test/fuzz/crypto_diff_fuzz_chacha20.cpp \ + test/fuzz/crypto_diff_fuzz_pychacha20.cpp \ test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp \ test/fuzz/crypto_poly1305.cpp \ test/fuzz/cuckoocache.cpp \ diff --git a/src/test/fuzz/crypto_diff_fuzz_pychacha20.cpp b/src/test/fuzz/crypto_diff_fuzz_pychacha20.cpp new file mode 100644 index 000000000000..82d035c9e29b --- /dev/null +++ b/src/test/fuzz/crypto_diff_fuzz_pychacha20.cpp @@ -0,0 +1,222 @@ +// Copyright (c) 2020-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SV_SOCK_PATH "/tmp/socket_test.s" + +// read size from a file descriptor +// return true if val is set, false for EOF +static bool read_uint32(int read_fd, uint32_t& val) +{ + unsigned char msgSizeBuf[4]; + unsigned iBuf = 0; + + while (iBuf < sizeof(msgSizeBuf)) { + ssize_t rc = read(read_fd, msgSizeBuf + iBuf, sizeof(msgSizeBuf) - iBuf); + + if (rc == 0) { + return false; + } else if (rc < 0) { + std::cout << __func__ << "@" << __LINE__ << ":::Read ERROR" << std::endl; + } else { + iBuf += rc; + } + } + + val = *(static_cast(static_cast(&msgSizeBuf[0]))); + + return true; +} + +// read message from a file descriptor +static std::vector read_string(int read_fd, uint32_t sz) +{ + std::vector msgBuf(sz); + unsigned iBuf = 0; + + while (iBuf < sz) { + ssize_t rc = ::read(read_fd, &(msgBuf[0]) + iBuf, sz - iBuf); + if (rc == 0) { + std::cout << __func__ << "@" << __LINE__ << ":::EOF read" << std::endl; + exit(1); + } else if (rc < 0) { + std::cout << __func__ << "@" << __LINE__ << ":::Read ERROR during message" << std::endl; + exit(1); + } else { + iBuf += rc; + } + } + return msgBuf; +} + +// read from file descriptor [size][message] +std::vector read_from_python(int sockfd) +{ + uint32_t apiArgSize; + if (!read_uint32(sockfd, apiArgSize)) { + std::cout << "EOF white reading apiArgSize" << std::endl; + ::exit(1); + } + std::vector apiArg = read_string(sockfd, apiArgSize); + return apiArg; +} + +// send [msg_size][message] to a file descriptor +static void send_msg(int write_fd, std::string msg) +{ + uint32_t msgSize = msg.size(); + unsigned char msgSizeBuf[4]; + memcpy(msgSizeBuf, &msgSize, sizeof(msgSize)); + unsigned iBuf = 0; + while (iBuf < 4) { + ssize_t rc = ::write(write_fd, msgSizeBuf + iBuf, sizeof(msgSizeBuf) - iBuf); + if (rc < 0) { + std::cout << "Error writing message size" << std::endl; + } else if (rc == 0) { + std::cout << "rc == 0, what does that mean?" << std::endl; + exit(1); + } else { + iBuf += rc; + } + } + + iBuf = 0; + const char* msgBuf = msg.c_str(); + while (iBuf < msgSize) { + ssize_t rc = ::write(write_fd, msgBuf + iBuf, msgSize - iBuf); + if (rc < 0) { + std::cout << "Error writing message" << std::endl; + } else if (rc == 0) { + std::cout << "rc == 0, what does that mean?" << std::endl; + exit(1); + } else { + iBuf += rc; + } + } +} + +// send a string to the file descriptor +void send_to_python(int sockfd, std::string str) +{ + std::ostringstream os1; + os1 << str; + send_msg(sockfd, os1.str()); +} + +// send a number to the file descriptor +void send_to_python(int sockfd, uint32_t num) +{ + unsigned char msgSizeBuf[4]; + memcpy(msgSizeBuf, &num, sizeof(num)); + unsigned iBuf = 0; + while (iBuf < 4) { + ssize_t rc = ::write(sockfd, msgSizeBuf + iBuf, sizeof(msgSizeBuf) - iBuf); + if (rc < 0) { + std::cout << "Error writing message size" << std::endl; + } else if (rc == 0) { + std::cout << "rc == 0, what does that mean?" << std::endl; + exit(1); + } else { + iBuf += rc; + } + } +} + +FUZZ_TARGET(crypto_diff_fuzz_pychacha20) +{ + /* ----------------------- socket initialisation -------------------------- */ + std::cout << "socket initialisation\n"; + struct sockaddr_un addr; + + // Create a new client socket with domain: AF_UNIX, type: SOCK_STREAM, protocol: 0 + int sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + + // Make sure socket's file descriptor is legit. + if (sockfd == -1) { + std::cout << "socket creation failed...\n"; + exit(1); + } + + // Construct server address, and make the connection. + memset(&addr, 0, sizeof(struct sockaddr_un)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, SV_SOCK_PATH, sizeof(addr.sun_path) - 1); + + struct timeval tv; + tv.tv_sec = 60; + tv.tv_usec = 0; + setsockopt(sockfd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof tv); + + // Connects the active socket(sockfd) to the listening socket whose address is addr. + if (connect(sockfd, (struct sockaddr*)&addr, sizeof(struct sockaddr_un)) == -1) { + std::cout << "connection with the server failed...\n"; + exit(1); + } + /* ----------------------- initialisation over ----------------------- */ + std::cout << "socket initialisation over\n"; + /* ----------------------- fuzzing phase ------------------------------ */ + std::cout << "fuzzing initialisation\n"; + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + ChaCha20 chacha20; + + const std::vector key = ConsumeFixedLengthByteVector(fuzzed_data_provider, 32); + chacha20 = ChaCha20{key.data(), key.size()}; + // send to python server [4][init][key.size][key.data] + send_to_python(sockfd, "init"); + send_to_python(sockfd, std::string(key.begin(), key.end())); + // check if response from python server is "ok" + std::vector response = read_from_python(sockfd); + std::string s1(response.begin(), response.end()); + assert(s1 == "ok"); + std::cout<<"py response obtained\n"; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 3000) + { + CallOneOf( + fuzzed_data_provider, + [&] { + uint32_t integralInRange = fuzzed_data_provider.ConsumeIntegralInRange(0, 4096); + std::vector cpp_keystream(integralInRange); + chacha20.Keystream(cpp_keystream.data(), cpp_keystream.size()); + // send to python server [6]["stream"][keystream_size] + send_to_python(sockfd, "stream"); + send_to_python(sockfd, integralInRange); + // check if the cpp and python computations of the keystream match + std::vector py_keystream = read_from_python(sockfd); + assert(cpp_keystream == py_keystream); + }, + [&] { + uint32_t integralInRange = fuzzed_data_provider.ConsumeIntegralInRange(0, 4096); + std::vector cpp_ciphertext(integralInRange); + const std::vector plaintext = ConsumeFixedLengthByteVector(fuzzed_data_provider, integralInRange); + chacha20.Crypt(plaintext.data(), cpp_ciphertext.data(), integralInRange); + // send to python server [5]["crypt"][size][plaintext] + send_to_python(sockfd, "crypt"); + send_to_python(sockfd, std::string(plaintext.begin(), plaintext.end())); + // check if the cpp and python ciphertext match + std::vector py_ciphertext = read_from_python(sockfd); + assert(cpp_ciphertext == py_ciphertext); + }); + } + send_to_python(sockfd, "exit"); + response = read_from_python(sockfd); + std::string s2(response.begin(), response.end()); + assert(s2 == "ok"); + /* ----------------------- end fuzzing phase ----------------------- */ + std::cout << "fuzzing over\n"; + close(sockfd); +} \ No newline at end of file diff --git a/src/test/fuzz/script.py b/src/test/fuzz/script.py new file mode 100644 index 000000000000..39cf3bd85b3b --- /dev/null +++ b/src/test/fuzz/script.py @@ -0,0 +1,75 @@ +import os +import socket +import struct +import sys + +dirname = os.path.dirname(__file__) +filename = os.path.join(dirname, '../../../test/functional/test_framework') +sys.path.insert(0, filename) +from chacha20 import ChaCha20 + +if os.path.exists("/tmp/socket_test.s"): + os.remove("/tmp/socket_test.s") + +def main(): + print("Script Hello!!!") + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + s.bind("/tmp/socket_test.s") + s.listen(1) + while True: + chacha = None + conn, addr = s.accept() + with conn: + while True: + # get [size][cmd] from client about operation to be performed + req = conn.recv(4) + cmd_size = struct.unpack('> (32 - c)) + + @staticmethod + def quarter_round(x, a, b, c, d): + """Perform a ChaCha quarter round""" + xa = x[a] + xb = x[b] + xc = x[c] + xd = x[d] + + xa = (xa + xb) & 0xffffffff + xd = xd ^ xa + xd = ((xd << 16) & 0xffffffff | (xd >> 16)) + + xc = (xc + xd) & 0xffffffff + xb = xb ^ xc + xb = ((xb << 12) & 0xffffffff | (xb >> 20)) + + xa = (xa + xb) & 0xffffffff + xd = xd ^ xa + xd = ((xd << 8) & 0xffffffff | (xd >> 24)) + + xc = (xc + xd) & 0xffffffff + xb = xb ^ xc + xb = ((xb << 7) & 0xffffffff | (xb >> 25)) + + x[a] = xa + x[b] = xb + x[c] = xc + x[d] = xd + + _round_mixup_box = [(0, 4, 8, 12), + (1, 5, 9, 13), + (2, 6, 10, 14), + (3, 7, 11, 15), + (0, 5, 10, 15), + (1, 6, 11, 12), + (2, 7, 8, 13), + (3, 4, 9, 14)] + + @classmethod + def double_round(cls, x): + """Perform two rounds of ChaCha cipher""" + for a, b, c, d in cls._round_mixup_box: + xa = x[a] + xb = x[b] + xc = x[c] + xd = x[d] + + xa = (xa + xb) & 0xffffffff + xd = xd ^ xa + xd = ((xd << 16) & 0xffffffff | (xd >> 16)) + + xc = (xc + xd) & 0xffffffff + xb = xb ^ xc + xb = ((xb << 12) & 0xffffffff | (xb >> 20)) + + xa = (xa + xb) & 0xffffffff + xd = xd ^ xa + xd = ((xd << 8) & 0xffffffff | (xd >> 24)) + + xc = (xc + xd) & 0xffffffff + xb = xb ^ xc + xb = ((xb << 7) & 0xffffffff | (xb >> 25)) + + x[a] = xa + x[b] = xb + x[c] = xc + x[d] = xd + + @staticmethod + def chacha_block(key, counter, nonce, rounds): + """Generate a state of a single block""" + counter = bytearray(counter.to_bytes(8, sys.byteorder)) + state = ChaCha20.constants + key + ChaCha20._bytearray_to_words(counter) + nonce + + working_state = state[:] + dbl_round = ChaCha20.double_round + for _ in range(0, rounds // 2): + dbl_round(working_state) + + return [(st + wrkSt) & 0xffffffff for st, wrkSt in zip(state, working_state)] + + @staticmethod + def word_to_bytearray(state): + """Convert state to little endian bytestream""" + return bytearray(struct.pack(' 0: + encrypted_message += bytearray(x ^ y for x, y in zip(self.keystream_bytes[self.keystream_next_index:], block[:bytes_left_prev_keystream])) + self.counter += 1 + self.keystream_bytes = self.key_stream() + self.keystream_next_index = 0 + else: + self.counter += 1 + self.keystream_bytes = self.key_stream() + self.keystream_next_index = 0 + encrypted_message += bytearray(x ^ y for x, y in zip(self.keystream_bytes, block)) + return encrypted_message + + def key_stream(self): + """receive the key stream""" + key_stream = ChaCha20.chacha_block(self.key, + self.counter, + self.nonce, + self.rounds) + key_stream = ChaCha20.word_to_bytearray(key_stream) + return key_stream + + def decrypt(self, ciphertext): + """Decrypt the data""" + return self.encrypt(ciphertext) \ No newline at end of file diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index e2eab2a0feca..e0de7bc9fe22 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -134,6 +134,14 @@ def main(): logging.info("Please consider adding a fuzz corpus at https://github.com/bitcoin-core/qa-assets") try: + if 'crypto_diff_fuzz_pychacha20' in args.target: + # # output += "hi" + theproc =subprocess.Popen('src/test/fuzz/script.py', shell = True) + theproc.communicate() + import time; time.sleep(2) + # check if socket is still connected + # # sleep(10) + # # output += "bye" help_output = subprocess.run( args=[ os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', 'fuzz'), @@ -217,7 +225,7 @@ def job(command, t): for future in as_completed(futures): future.result() - +# todo: what's this? def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, build_dir, merge_dir): logging.info("Merge the inputs from the passed dir into the corpus_dir. Passed dir {}".format(merge_dir)) jobs = [] @@ -266,6 +274,12 @@ def run_once(*, fuzz_pool, corpus, test_list, src_dir, build_dir, use_valgrind): def job(t, args): output = 'Run {} with args {}'.format(t, args) + # import ipdb; ipdb.set_trace() + # if t == 'crypto_diff_fuzz_pychacha20': + # output += "hi" + # subprocess.Popen(['python','src/test/fuzz/script.py']) + # sleep(10) + # output += "bye" result = subprocess.run( args, env=get_fuzz_env(target=t, source_dir=src_dir),