diff --git a/.dockerignore b/.dockerignore index e083811db..2d71ff065 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,7 +13,8 @@ charts/ # Prevent copying scripts that are unused in docker scripts/ !scripts/test.sh -!scripts/configure.sh +!scripts/install-build-tools.sh +!scripts/setup-dependencies.sh !scripts/test-transaction.sh !scripts/wait-for-it.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6efc008d0..cf85442c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,10 @@ jobs: - uses: actions/checkout@v2 with: submodules: recursive - - name: Configure - run: sudo ./scripts/configure.sh + - name: Setup Build Env + run: sudo ./scripts/install-build-tools.sh + - name: Setup Local Dependencies + run: ./scripts/setup-dependencies.sh - name: Build run: scripts/build.sh lint: @@ -36,8 +38,10 @@ jobs: - uses: actions/checkout@v2 with: submodules: recursive - - name: Configure - run: sudo ./scripts/configure.sh + - name: Setup Build Env + run: sudo ./scripts/install-build-tools.sh + - name: Setup Local Dependencies + run: ./scripts/setup-dependencies.sh - name: Build run: scripts/build.sh - name: Lint @@ -50,8 +54,10 @@ jobs: - uses: actions/checkout@v2 with: submodules: recursive - - name: Configure - run: sudo ./scripts/configure.sh + - name: Setup Build Env + run: sudo ./scripts/install-build-tools.sh + - name: Setup Local Dependencies + run: ./scripts/setup-dependencies.sh - name: Build run: scripts/build.sh - name: Run Unit Tests diff --git a/.gitignore b/.gitignore index 433657a75..ca0adf668 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ doxygen_generated # Default build folder build/ +# Default dependency installation folder +prefix/ + # 3rd party generated files 3rdparty/secp256k1_genctx 3rdparty/src/ecmult_static_context.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 2dd8d679f..1e8f84695 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,6 +49,10 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") include_directories(3rdparty 3rdparty/secp256k1/include /usr/lib /usr/local/lib /usr/local/include /opt/homebrew/include) +if(DEFINED CMAKE_PREFIX_PATH) + include_directories(SYSTEM "${CMAKE_PREFIX_PATH}/include") +endif() + if(CMAKE_BUILD_TYPE STREQUAL "Debug") add_compile_options(-fprofile-arcs -ftest-coverage) endif() diff --git a/Dockerfile b/Dockerfile index 5ee30d0b8..a7c3f934b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,12 +14,14 @@ ENV BUILD_RELEASE 1 RUN mkdir -p /opt/tx-processor/scripts -COPY scripts/configure.sh /opt/tx-processor/scripts/configure.sh +COPY scripts/install-build-tools.sh /opt/tx-processor/scripts/install-build-tools.sh +COPY scripts/setup-dependencies.sh /opt/tx-processor/scripts/setup-dependencies.sh # Set working directory WORKDIR /opt/tx-processor -RUN scripts/configure.sh +RUN scripts/install-build-tools.sh +RUN scripts/setup-dependencies.sh # Create Build Image FROM $BASE_IMAGE AS builder @@ -30,7 +32,7 @@ COPY . . # Build binaries RUN mkdir build && \ cd build && \ - cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} .. && \ + cmake -DCMAKE_PREFIX_PATH="prefix" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} .. && \ make -j$(nproc) # Create 2PC Deployment Image diff --git a/README.md b/README.md index 3dee15f7c..7b6162ce0 100644 --- a/README.md +++ b/README.md @@ -69,24 +69,33 @@ If you want to dive straight in, take a look at our issue tracker's list of [goo 1. Clone the repository (including submodules) - `git clone --recurse-submodules https://github.com/mit-dci/opencbdc-tx` -# Build +# Setup the build envirnoment Use these directions if you want to build the source on your machine. If you just want to run the system, see "Run the Code" below. -1. Install the necessary libraries and resources - ```terminal - # ./scripts/configure.sh - ``` +1. Setup the build-environment. + Note that this script is just a convenience to install system-wide dependencies we expect. + As a result, it uses the system package manager, requires `sudo`, and should only be run **once**. + ```console + # ./scripts/install-build-tools.sh + ``` +1. Setup project dependencies + This script builds and installs a local copy of several build-dependencies which are not widely packaged. + Because it installs to a local, configurable prefix (defaulting to `./prefix`), it does not need root permissions to run. + Furthermore, it should always be safe to delete `prefix` and rerun this script. + ```console + $ ./scripts/setup-dependencies.sh + ``` 1. Run the build - ```terminal - # ./scripts/build.sh - ``` + ```console + $ ./scripts/build.sh + ``` ## macOS Note that if you have not already installed the xcode cli tools you will need to: -```terminal +```console # xcode-select --install ``` # Run the Code @@ -100,11 +109,11 @@ See the [PArSEC User Guide](docs/parsec_user_guide.md) Running Unit & Integration Tests 1. Build all docker images - ```terminal + ```console $ ./scripts/build-docker.sh ``` 1. Run Unit & Integration Tests - ```terminal + ```console $ docker run -ti opencbdc-tx-builder ./scripts/test.sh ``` diff --git a/scripts/build.sh b/scripts/build.sh index ccfbfac97..6a1a2f158 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -3,6 +3,9 @@ set -e echo "Building..." +# see PREFIX in ./scripts/configure.sh +PREFIX="$(cd "$(dirname "$0")"/.. && pwd)/prefix" + if [ -z ${BUILD_DIR+x} ]; then export BUILD_DIR=build fi @@ -10,14 +13,14 @@ fi mkdir -p $BUILD_DIR cd $BUILD_DIR -CMAKE_FLAGS="" +CMAKE_FLAGS=-DCMAKE_PREFIX_PATH="${PREFIX}" CPUS=1 if [[ "$OSTYPE" == "linux-gnu"* ]]; then CPUS=$(grep -c ^processor /proc/cpuinfo) elif [[ "$OSTYPE" == "darwin"* ]]; then CPUS=$(sysctl -n hw.ncpu) XCODE_CMDLINE_DIR=$(xcode-select -p) - CMAKE_FLAGS+="-DCMAKE_C_COMPILER=${XCODE_CMDLINE_DIR}/usr/bin/clang -DCMAKE_CXX_COMPILER=${XCODE_CMDLINE_DIR}/usr/bin/clang++ -DCMAKE_CXX_FLAGS=-isystem\ /usr/local/include -DCMAKE_EXPORT_COMPILE_COMMANDS=ON" + CMAKE_FLAGS+=" -DCMAKE_C_COMPILER=${XCODE_CMDLINE_DIR}/usr/bin/clang -DCMAKE_CXX_COMPILER=${XCODE_CMDLINE_DIR}/usr/bin/clang++ -DCMAKE_CXX_FLAGS=-isystem\ /usr/local/include -DCMAKE_EXPORT_COMPILE_COMMANDS=ON" fi CMAKE_BUILD_TYPE="Debug" diff --git a/scripts/install-build-tools.sh b/scripts/install-build-tools.sh new file mode 100755 index 000000000..5a96b4286 --- /dev/null +++ b/scripts/install-build-tools.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +echo "Setting up build environment..." + +green="\033[0;32m" +cyan="\033[0;36m" +end="\033[0m" + +set -e + +SUDO='' +if (( $EUID != 0 )); then + echo -e "non-root user, sudo required" + SUDO='sudo' +fi + +if [[ "$OSTYPE" == "darwin"* ]]; then + CPUS=$(sysctl -n hw.ncpu) + # ensure development environment is set correctly for clang + $SUDO xcode-select -switch /Library/Developer/CommandLineTools + brew install llvm@14 googletest google-benchmark lcov make wget cmake curl + CLANG_TIDY=/usr/local/bin/clang-tidy + if [ ! -L "$CLANG_TIDY" ]; then + $SUDO ln -s $(brew --prefix)/opt/llvm@14/bin/clang-tidy /usr/local/bin/clang-tidy + fi + GMAKE=/usr/local/bin/gmake + if [ ! -L "$GMAKE" ]; then + $SUDO ln -s $(xcode-select -p)/usr/bin/gnumake /usr/local/bin/gmake + fi +fi + +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + $SUDO apt update + $SUDO apt install -y build-essential wget cmake libgtest-dev libbenchmark-dev lcov git software-properties-common rsync unzip + + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | $SUDO apt-key add - + $SUDO add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" + $SUDO apt install -y clang-format-14 clang-tidy-14 + $SUDO ln -s -f $(which clang-format-14) /usr/local/bin/clang-format + $SUDO ln -s -f $(which clang-tidy-14) /usr/local/bin/clang-tidy +fi + +PYTHON_TIDY=/usr/local/bin/run-clang-tidy.py +if [ ! -f "${PYTHON_TIDY}" ]; then + echo -e "${green}Copying run-clang-tidy to /usr/local/bin${end}" + wget https://raw.githubusercontent.com/llvm/llvm-project/e837ce2a32369b2e9e8e5d60270c072c7dd63827/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py + $SUDO mv run-clang-tidy.py /usr/local/bin +fi diff --git a/scripts/configure.sh b/scripts/setup-dependencies.sh similarity index 60% rename from scripts/configure.sh rename to scripts/setup-dependencies.sh index 5cb67d807..1e20e131b 100755 --- a/scripts/configure.sh +++ b/scripts/setup-dependencies.sh @@ -1,8 +1,6 @@ #!/bin/bash -echo "Configuring..." - -echo "test local pipeline changes" +echo "Setting up dependencies..." green="\033[0;32m" cyan="\033[0;36m" @@ -10,11 +8,13 @@ end="\033[0m" set -e -SUDO='' -if (( $EUID != 0 )); then - echo -e "non-root user, sudo required" - SUDO='sudo' -fi +# install in a custom prefix rather than /usr/local. by default, this +# chooses "prefix" directory alongside "scripts" where configure.sh +# resides. + +PREFIX="$(cd "$(dirname "$0")"/.. && pwd)/prefix" +echo "Will install local dependencies in the following prefix: $PREFIX" +mkdir -p $PREFIX $PREFIX/lib $PREFIX/include CMAKE_BUILD_TYPE="Debug" if [[ "$BUILD_RELEASE" == "1" ]]; then @@ -26,28 +26,6 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then CPUS=$(grep -c ^processor /proc/cpuinfo) elif [[ "$OSTYPE" == "darwin"* ]]; then CPUS=$(sysctl -n hw.ncpu) - # ensure development environment is set correctly for clang - $SUDO xcode-select -switch /Library/Developer/CommandLineTools - brew install llvm@14 googletest google-benchmark lcov make wget cmake curl - CLANG_TIDY=/usr/local/bin/clang-tidy - if [ ! -L "$CLANG_TIDY" ]; then - $SUDO ln -s $(brew --prefix)/opt/llvm@14/bin/clang-tidy /usr/local/bin/clang-tidy - fi - GMAKE=/usr/local/bin/gmake - if [ ! -L "$GMAKE" ]; then - $SUDO ln -s $(xcode-select -p)/usr/bin/gnumake /usr/local/bin/gmake - fi -fi - -if [[ "$OSTYPE" == "linux-gnu"* ]]; then - apt update - apt install -y build-essential wget cmake libgtest-dev libbenchmark-dev lcov git software-properties-common rsync unzip - - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | $SUDO apt-key add - - $SUDO add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" - $SUDO apt install -y clang-format-14 clang-tidy-14 - $SUDO ln -s -f $(which clang-format-14) /usr/local/bin/clang-format - $SUDO ln -s -f $(which clang-tidy-14) /usr/local/bin/clang-tidy fi LEVELDB_VERSION="1.23" @@ -58,9 +36,9 @@ tar xzvf ${LEVELDB_VERSION}.tar.gz rm -rf ${LEVELDB_VERSION}.tar.gz mv leveldb-${LEVELDB_VERSION} "leveldb-${LEVELDB_VERSION}-${CMAKE_BUILD_TYPE}" cd "leveldb-${LEVELDB_VERSION}-${CMAKE_BUILD_TYPE}" -cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DLEVELDB_BUILD_TESTS=0 -DLEVELDB_BUILD_BENCHMARKS=0 -DBUILD_SHARED_LIBS=0 -DHAVE_SNAPPY=0 . +cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DLEVELDB_BUILD_TESTS=0 -DLEVELDB_BUILD_BENCHMARKS=0 -DBUILD_SHARED_LIBS=0 -DHAVE_SNAPPY=0 . make -j$CPUS -$SUDO make install +make install cd .. NURAFT_VERSION="1.3.0" @@ -82,29 +60,22 @@ if [[ "$BUILD_RELEASE" == "1" ]]; then fi mkdir -p build cd build -cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DDISABLE_SSL=1 .. +cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DDISABLE_SSL=1 .. make -j$CPUS static_lib -echo -e "${green}Copying nuraft to /usr/local" -$SUDO cp libnuraft.a /usr/local/lib -$SUDO cp -r ../include/libnuraft /usr/local/include +echo -e "${green}Copying nuraft to $PREFIX/lib and $PREFIX/include${end}" +cp libnuraft.a $PREFIX/lib +cp -r ../include/libnuraft $PREFIX/include cd ../.. -PYTHON_TIDY=/usr/local/bin/run-clang-tidy.py -if [ ! -f "${PYTHON_TIDY}" ]; then - echo -e "${green}Copying run-clang-tidy to /usr/local/bin" - wget https://raw.githubusercontent.com/llvm/llvm-project/e837ce2a32369b2e9e8e5d60270c072c7dd63827/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py - $SUDO mv run-clang-tidy.py /usr/local/bin -fi - wget https://www.lua.org/ftp/lua-5.4.3.tar.gz rm -rf lua-5.4.3 tar zxf lua-5.4.3.tar.gz rm -rf lua-5.4.3.tar.gz cd lua-5.4.3 make -j$CPUS -$SUDO make install +make INSTALL_TOP=$PREFIX install cd .. if [[ "$OSTYPE" != "darwin"* ]]; then @@ -116,9 +87,9 @@ if [[ "$OSTYPE" != "darwin"* ]]; then rm -rf curl-${CURL_VERSION}.tar.gz mkdir -p curl-${CURL_VERSION}/build cd curl-${CURL_VERSION}/build - ../configure --disable-shared --without-ssl --without-libpsl --without-libidn2 --without-brotli --without-zstd --without-zlib + ../configure --prefix="${PREFIX}" --disable-shared --without-ssl --without-libpsl --without-libidn2 --without-brotli --without-zstd --without-zlib make -j$CPUS - $SUDO make install + make install cd ../.. fi @@ -129,9 +100,9 @@ tar xzvf ${JSONCPP_VERSION}.tar.gz rm -rf ${JSONCPP_VERSION}.tar.gz mkdir -p jsoncpp-${JSONCPP_VERSION}/build cd jsoncpp-${JSONCPP_VERSION}/build -cmake .. -DBUILD_SHARED_LIBS=NO -DBUILD_STATIC_LIBS=YES -DJSONCPP_WITH_TESTS=OFF -DJSONCPP_WITH_POST_BUILD_UNITTEST=OFF +cmake .. -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DBUILD_SHARED_LIBS=NO -DBUILD_STATIC_LIBS=YES -DJSONCPP_WITH_TESTS=OFF -DJSONCPP_WITH_POST_BUILD_UNITTEST=OFF make -j$CPUS -$SUDO make install +make install cd ../.. # NOTE: evmc v10.0.0 requires evmone v0.9.0 @@ -144,9 +115,9 @@ rm v${EVMC_VER}.zip cd evmc-${EVMC_VER} mkdir build cd build -cmake .. +cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" .. make -j$CPUS -$SUDO make install +make install cd ../.. # NOTE: updating evmone to v0.10.0 requires c++20 @@ -161,13 +132,13 @@ mv ../evmc-${EVMC_VER} ./evmc mkdir ./evmc/.git if [[ "$OSTYPE" == "darwin"* ]]; then # Mac Silicon: clang 'ar' does not allow empty member list, fails w/ -DBUILD_SHARED_LIBS=OFF - cmake -S . -B build + cmake -S . -B build -DCMAKE_INSTALL_PREFIX="${PREFIX}" else - cmake -S . -B build -DBUILD_SHARED_LIBS=OFF + cmake -S . -B build -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX="${PREFIX}" fi cmake --build build --parallel cd build -$SUDO make install +make install cd ../.. rm -rf evmone-${EVMONE_VER} @@ -180,8 +151,8 @@ mkdir build cd build cmake -DETHASH_BUILD_ETHASH=OFF -DETHASH_BUILD_TESTS=OFF .. cmake --build . --parallel -$SUDO cp ./lib/keccak/libkeccak.a /usr/local/lib -$SUDO cp -r ../include/ethash /usr/local/include +cp ./lib/keccak/libkeccak.a $PREFIX/lib +cp -r ../include/ethash $PREFIX/include cd ../.. wget https://gnu.askapache.com/libmicrohttpd/libmicrohttpd-0.9.75.tar.gz @@ -191,8 +162,8 @@ rm libmicrohttpd-0.9.75.tar.gz cd libmicrohttpd-0.9.75 mkdir build cd build -../configure --disable-curl --disable-examples --disable-doc --disable-shared --disable-https +../configure --prefix="${PREFIX}" --disable-curl --disable-examples --disable-doc --disable-shared --disable-https make -j $CPUS -$SUDO make install +make install cd ../../ rm -rf libmicrohttpd-0.9.75