Introduce merkle tree implementation

.clangd

@@ -0,0 +1,2 @@
+CompileFlags:
+  Add: [-D__cpp_concepts=202002L]

.gitignore

@@ -16,10 +16,6 @@
 *.dylib
 *.dll
 
-# Fortran module files
-*.mod
-*.smod
-
 # Compiled Static libraries
 *.lai
 *.la
@@ -34,10 +30,11 @@
 # IDE auxiliaries
 .idea
 
-/build
+/build*
 /cmake-build-*
-
 /.build
 /.venv
 /.vcpkg
 /.build*
+
+CMakeUserPresets.json

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "clangd.arguments": [
+        "--compile-commands-dir=build-clang-20-debug"
+    ]
+}

CMakeLists.txt

@@ -6,54 +6,108 @@
 
 cmake_minimum_required(VERSION 3.25)
 
-option(TESTING "Build and run test suite" ON)
-if (TESTING)
-  list(APPEND VCPKG_MANIFEST_FEATURES test)
-endif ()
+option(MORUM_BUILD_TESTS "Build tests" ON)
+option(MORUM_BUILD_BENCHMARKS "Build benchmarks" ON)
+option(MORUM_BUILD_TRACY "Enable Tracy profiler" OFF)
+
+if (MORUM_BUILD_TESTS)
+    list(APPEND VCPKG_MANIFEST_FEATURES "test")
+endif()
+if (MORUM_BUILD_BENCHMARKS)
+    list(APPEND VCPKG_MANIFEST_FEATURES "benchmark")
+endif()
+if (MORUM_BUILD_TRACY)
+    list(APPEND VCPKG_MANIFEST_FEATURES "tracy")
+endif()
 
-set(CMAKE_CXX_STANDARD 20)
+project(morum
+    VERSION 0.1.0
+    LANGUAGES C CXX
+)
+set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
-project(cpp-jam
-    VERSION 0.0.1
-    LANGUAGES CXX
-)
-
-message(STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
-message(STATUS "Boost_DIR: ${Boost_DIR}")
+add_compile_options(-Wall -Wextra)
 
-find_package(Python3 REQUIRED)
+if ((CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+    add_compile_options(-flto=thin)
+endif()
 
-find_package(PkgConfig REQUIRED)
-pkg_check_modules(libb2 REQUIRED IMPORTED_TARGET GLOBAL libb2)
+option(MORUM_ASAN "Enable address sanitizer" OFF)
+option(MORUM_TSAN "Enable address sanitizer" OFF)
+option(MORUM_UBSAN "Enable address sanitizer" OFF)
+option(MORUM_TRACE "Enable tracing" OFF)
 
-find_package(Boost CONFIG REQUIRED COMPONENTS algorithm outcome program_options)
+find_package(Python3 REQUIRED)
+find_package(Boost CONFIG REQUIRED COMPONENTS algorithm program_options outcome)
 find_package(fmt CONFIG REQUIRED)
 find_package(yaml-cpp CONFIG REQUIRED)
-find_package(jam_crust CONFIG REQUIRED)
 find_package(scale CONFIG REQUIRED)
 find_package(soralog CONFIG REQUIRED)
-find_package(schnorrkel_crust CONFIG REQUIRED)
 find_package(Boost.DI CONFIG REQUIRED)
 find_package(qtils CONFIG REQUIRED)
 find_package(prometheus-cpp CONFIG REQUIRED)
+find_package(RocksDB CONFIG REQUIRED)
+find_package(nlohmann_json CONFIG REQUIRED)
+find_package(qdrvm-crates CONFIG REQUIRED)
+
+find_library(BLAKE2_LIB blake2b)
+add_library(blake2b STATIC IMPORTED)
+set_property(TARGET blake2b PROPERTY IMPORTED_LOCATION ${BLAKE2_LIB})
 
 add_library(headers INTERFACE)
 target_include_directories(headers INTERFACE
     $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src_>
     $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
 )
 
+if (MORUM_ASAN)
+    add_compile_options(-fsanitize=address)
+    add_link_options(-fsanitize=address)
+endif()
+
+if (MORUM_UBSAN)
+    add_compile_options(-fsanitize=undefined -fno-sanitize-recovery=undefined)
+    add_link_options(-fsanitize=undefined)
+endif()
+if (MORUM_TSAN)
+    add_compile_options(-fsanitize=thread)
+    add_link_options(-fsanitize=thread)
+endif()
+
+if (CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+    option(QTILS_ASSERT "Enable asserts" OFF)
+else()
+    option(QTILS_ASSERT "Enable asserts" ON)
+endif()
+
 add_subdirectory(src)
 
-if (TESTING)
-  enable_testing()
+if(MORUM_BUILD_BENCHMARKS)
+    find_package(benchmark CONFIG REQUIRED)
+    add_subdirectory(benchmark)
+endif()
 
-  find_package(GTest CONFIG REQUIRED)
-  set(GTEST_DEPS GTest::gtest_main)
+if(MORUM_BUILD_TESTS)
+    find_package(GTest CONFIG REQUIRED)
+    set(GTEST_DEPS GTest::gtest_main)
+
+    enable_testing()
+
+    function (morum_add_test TEST_NAME TEST_SRC)
+        add_executable(${TEST_NAME} ${TEST_SRC})
+        target_include_directories(${TEST_NAME} 
+            PUBLIC 
+            ${CMAKE_SOURCE_DIR}/include
+        )
+        add_compile_definitions(${TEST_NAME} PUBLIC MORUM_TEST_BUILD)
+        add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME} WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/test_bin")
+        set_target_properties(${TEST_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/test_bin")
+    endfunction()
 
-  add_subdirectory(test-vectors)
-  add_subdirectory(tests)
+    add_subdirectory(test-vectors)
+    add_subdirectory(test)
 endif ()

CMakePresets.json

@@ -2,13 +2,38 @@
   "version": 2,
   "configurePresets": [
     {
-      "name": "default",
+      "name": "vcpkg",
+      "hidden": true,
       "generator": "Ninja",
-      "binaryDir": "${sourceDir}/build",
       "cacheVariables": {
-        "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
-        "CMAKE_BUILD_TYPE": "Debug"
+        "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
+      }
+    },
+    {
+      "name": "dev",
+      "inherits": "vcpkg",
+      "binaryDir": "${sourceDir}/build-dev",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo",
+        "QTILS_ASSERT": "ON",
+        "MORUM_BUILD_TRACY": "ON",
+        "MORUM_ASAN": "OFF",
+        "MORUM_UBSAN": "OFF",
+        "MORUM_TSAN": "OFF"
+      }
+    },
+    {
+      "name": "debug",
+      "inherits": "vcpkg",
+      "binaryDir": "${sourceDir}/build-debug",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "MORUM_BUILD_TRACY": "ON",
+        "MORUM_BUILD_TESTS": "ON",
+        "MORUM_ASAN": "OFF",
+        "MORUM_UBSAN": "OFF",
+        "MORUM_TSAN": "OFF"
       }
     }
   ]
-}
+}

CODESTYLE.md

@@ -15,7 +15,7 @@
 #include <boost/...>
 #include <fmt/...>
 #include <gtest/...>
-#include <jam_crust.h>
+#include <ark_vrf/...>
 #include <qtils/...>
 #include <scale/...>
 

Makefile

@@ -36,7 +36,7 @@ init_vcpkg:
 
 configure:
 	@echo "=== Configuring..."
-	VCPKG_ROOT=$(VCPKG) cmake --preset=default -DPython3_EXECUTABLE="$(VENV)/bin/python3" -B $(BUILD) $(PROJECT)
+	VCPKG_ROOT=$(VCPKG) cmake --preset=debug -DPython3_EXECUTABLE="$(VENV)/bin/python3" -B $(BUILD) $(PROJECT)
 
 build:
 	@echo "=== Building..."

benchmark/CMakeLists.txt

@@ -0,0 +1,3 @@
+
+add_executable(set_get_benchmark merkle_tree/set_get.cpp)
+target_link_libraries(set_get_benchmark benchmark::benchmark merkle_tree)

benchmark/merkle_tree/set_get.cpp

@@ -0,0 +1,127 @@
+#include <cstddef>
+#include <expected>
+#include <filesystem>
+#include <memory>
+#include <optional>
+#include <random>
+#include <ranges>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+#include <client/TracyScoped.hpp>
+#include <tracy/Tracy.hpp>
+
+#include <morum/archive_backend.hpp>
+#include <morum/common.hpp>
+#include <morum/db.hpp>
+#include <morum/merkle_tree.hpp>
+
+constexpr unsigned seed = 42;
+static std::mt19937_64 rand_engine{seed};
+
+template <std::ranges::input_range R>
+void fill_random(R &&span) {
+  static std::uniform_int_distribution dist;
+
+  for (auto &byte : span) {
+    byte = dist(rand_engine);
+  }
+}
+
+morum::Hash32 random_hash() {
+  morum::Hash32 hash;
+  fill_random(hash);
+  return hash;
+}
+
+morum::ByteVector random_vector(size_t min_size = 1, size_t max_size = 128) {
+  std::uniform_int_distribution<size_t> dist(min_size, max_size);
+  size_t size = dist(rand_engine);
+
+  morum::ByteVector v(size);
+  fill_random(v);
+  return v;
+}
+
+std::unique_ptr<morum::ArchiveTrieDb> trie_db;
+morum::Hash32 last_root{};
+
+static void BM_SetGet(benchmark::State &state) {
+  rand_engine.seed(42);
+  constexpr int INSERTION_NUM = 1000;
+
+  for (auto _ : state) {
+    ZoneNamedN(loop_zone, "loop", true);
+
+    auto tree = trie_db->load_tree(last_root).value().value();
+
+    std::vector<std::pair<morum::Hash32, morum::ByteVector>> insertions;
+    for (int i = 0; i < INSERTION_NUM; i++) {
+      insertions.emplace_back(random_hash(), random_vector());
+    }
+    {
+      ZoneNamedN(setter_zone, "set", true);
+      for (auto &[k, v] : insertions) {
+        tree->set(k, morum::ByteVector{v}).value();
+      }
+    }
+    {
+      ZoneNamedN(getter_zone, "get", true);
+
+      for (auto &[k, v] : insertions) {
+        tree->get(k).value();
+      }
+    }
+    {
+      ZoneNamedN(calculate_hash_zone, "calculate_hash", true);
+      trie_db->get_root_and_store(*tree).value();
+    }
+
+    FrameMark;
+  }
+}
+
+BENCHMARK(BM_SetGet);
+
+template <typename F>
+struct FinalAction {
+  ~FinalAction() {
+    f();
+  }
+
+  F f;
+};
+
+int main(int argc, char **argv) {
+  char arg0_default[] = "benchmark";
+  char *args_default = arg0_default;
+  if (!argv) {
+    argc = 1;
+    argv = &args_default;
+  }
+
+  auto db = std::shared_ptr{morum::open_db("./test_db").value()};
+  FinalAction cleanup{[]() { std::filesystem::remove_all("./test_db"); }};
+
+  trie_db = std::make_unique<morum::ArchiveTrieDb>(db);
+
+  auto tree = trie_db->empty_tree();
+  constexpr int INSERTION_NUM = 5000;
+  {
+    ZoneNamedN(setter_zone, "initial insertions", true);
+
+    for (int i = 0; i < INSERTION_NUM; i++) {
+      tree->set(random_hash(), morum::ByteVector{random_vector()}).value();
+    }
+  }
+  last_root = trie_db->get_root_and_store(*tree).value();
+  ::benchmark::Initialize(&argc, argv);
+  if (::benchmark::ReportUnrecognizedArguments(argc, argv)) {
+    return 1;
+  }
+  ::benchmark::RunSpecifiedBenchmarks();
+  ::benchmark::Shutdown();
+  return 0;
+}