Have basics of a configuration generation tool for expanded testing. I foresee much more expanded functionality in the future and a lot of potential. This will require some possible rework of some existing code, however, as well as integration tests to make use of this proposed functionality. Also reorganized config files into their own directory which seems neater. Also needed to fix docker stff. Might need to revert docker stuff. Not sure how that will affect others.

Signed-off-by: Yiannis Karavas <[email protected]>

Author: ykaravas

CMakeLists.txt

@@ -75,5 +75,6 @@ endif()
 
 add_subdirectory(src)
 add_subdirectory(tests)
+add_subdirectory(tools/config_generator)
 add_subdirectory(tools/bench)
 add_subdirectory(tools/shard-seeder)

README.md

@@ -121,38 +121,38 @@ Additionally, you can start the atomizer architecture by passing `--file docker-
 ## Setup test wallets and test them
 
 The following commands are all performed from within the second container we started in the previous step.
-In each of the below commands, you should pass `atomizer-compose.cfg` instead of `2pc-compose.cfg` if you started the atomizer architecture.
+In each of the below commands, you should pass `config/general/atomizer-compose.cfg` instead of `config/general/2pc-compose.cfg` if you started the atomizer architecture.
 
 * Mint new coins (e.g., 10 new UTXOs each with a value of 5 atomic units of currency)
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat mint 10 5
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat mint 10 5
   [2021-08-17 15:11:57.686] [WARN ] Existing wallet file not found
   [2021-08-17 15:11:57.686] [WARN ] Existing mempool not found
   4bc23da407c3a8110145c5b6c38199c8ec3b0e35ea66bbfd78f0ed65304ce6fa
   ```
 
   If using the atomizer architecture, you'll need to sync the wallet after:
   ```terminal
-  # ./build/src/uhs/client/client-cli atomizer-compose.cfg mempool0.dat wallet0.dat sync
+  # ./build/src/uhs/client/client-cli config/general/atomizer-compose.cfg mempool0.dat wallet0.dat sync
   ```
 
 * Inspect the balance of a wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat info
   Balance: $0.50, UTXOs: 10, pending TXs: 0
   ```
 
 * Make a new wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat newaddress
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat newaddress
   [2021-08-17 15:13:16.148] [WARN ] Existing wallet file not found
   [2021-08-17 15:13:16.148] [WARN ] Existing mempool not found
   usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
   ```
 
 * Send currency from the first wallet to the second wallet created in the previous step (e.g., 30 atomic units of currency)
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat send 30 usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat send 30 usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
   tx_id:
   cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e75
   Data for recipient importinput:
@@ -162,20 +162,20 @@ In each of the below commands, you should pass `atomizer-compose.cfg` instead of
 
   If using the atomizer architecture, you'll need to sync the sending wallet after:
   ```terminal
-  # ./build/src/uhs/client/client-cli atomizer-compose.cfg mempool0.dat wallet0.dat sync
+  # ./build/src/uhs/client/client-cli config/general/atomizer-compose.cfg mempool0.dat wallet0.dat sync
   ```
 
 * Check that the currency is no longer available in the sending wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat info
   Balance: $0.20, UTXOs: 4, pending TXs: 0
   ```
 
 * Import coins to the receiving wallet using the string after `importinput` from the currency transfer step above
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat importinput cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e750000000000000000d0e4f689b550f623e9370edae235de50417860be0f2f8e924eca9f402fcefeaa1e00000000000000
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat sync
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat importinput cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e750000000000000000d0e4f689b550f623e9370edae235de50417860be0f2f8e924eca9f402fcefeaa1e00000000000000
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat sync
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat info
   Balance: $0.30, UTXOs: 1, pending TXs: 0
   ```
 

2pc-compose.cfg config/general/2pc-compose.cfg

@@ -0,0 +1,44 @@
+2pc=1
+sentinel_count=1
+shard_count=1
+coordinator_count=1
+coordinator_max_threads=1
+election_timeout_upper=4000
+election_timeout_lower=3000
+heartbeat=1000
+raft_max_batch=100000
+snapshot_distance=1000000000
+batch_size=1
+wait_for_followers=0
+loadgen_invalid_tx_rate=0.00
+loadgen_fixed_tx_rate=0.01
+loadgen_sendtx_output_count=1
+loadgen_sendtx_input_count=1
+initial_mint_count=20000
+initial_mint_value=100
+tmpl_randomize_values=1
+tmpl_shard_start=0
+tmpl_shard_size=255
+tmpl_max_shard_raft_replication_count=1
+tmpl_avg_shard_start_end_overlap_percent=0.15
+tmpl_max_coordinator_raft_replication_count=1
+tmpl_default_log_level="INFO"
+tmpl_universal_override_log_level="WARN"
+tmpl_sentinel_log_level="WARN"
+tmpl_coordinator_log_level="DEBUG"
+tmpl_shard_log_level="DEBUG"
+num_wallets=10
+num_minters=1
+num_redeemers=1
+total_number_of_transactions=100
+avg_mint_value=10
+avg_mint_count=10
+avg_redemption_value=5
+avg_redemption_count=5
+transaction_frequency=5
+mint_frequency=0.1
+redemption_frequency=0.05
+sentinel_offline_probability=0.01
+shard_offline_probability=0.01
+coordinator_offline_probability=0.01
+randomize_execution=0

2pc.cfg.sample config/general/2pc.cfg.sample

@@ -0,0 +1,32 @@
+2pc=0
+archiver_count=1
+atomizer_count=1
+shard_count=1
+sentinel_count=1
+watchtower_count=1
+target_block_interval=3000
+stxo_cache_depth=2
+target_block_interval=250
+election_timeout_upper=4000
+election_timeout_lower=3000
+heartbeat=1000
+raft_max_batch=100000
+snapshot_distance=1000000000
+batch_size=1
+wait_for_followers=0
+loadgen_invalid_tx_rate=0.00
+loadgen_fixed_tx_rate=0.01
+loadgen_sendtx_output_count=1
+loadgen_sendtx_input_count=1
+initial_mint_count=20000
+initial_mint_value=100
+tmpl_randomize_values=1
+tmpl_shard_start=0
+tmpl_shard_size=255
+tmpl_avg_shard_start_end_overlap_percent=0.15
+tmpl_default_log_level="INFO"
+tmpl_sentinel_log_level="DEBUG"
+tmpl_shard_log_level="INFO"
+tmpl_watchtower_log_level="DEBUG"
+tmpl_archiver_log_level="DEBUG"
+tmpl_atomizer_log_level="DEBUG"

atomizer-compose.cfg config/general/atomizer-compose.cfg

@@ -7,7 +7,7 @@ services:
     image: opencbdc-tx
     tty: true
     restart: always
-    command: ./build/src/uhs/twophase/sentinel_2pc/sentineld-2pc 2pc-compose.cfg 0
+    command: ./build/src/uhs/twophase/sentinel_2pc/sentineld-2pc ./config/general/2pc-compose.cfg 0
     ports:
       - 5555:5555
     depends_on:
@@ -25,7 +25,7 @@ services:
     build: .
     image: opencbdc-tx
     tty: true
-    command: ./build/src/uhs/twophase/coordinator/coordinatord 2pc-compose.cfg 0 0
+    command: ./build/src/uhs/twophase/coordinator/coordinatord ./config/general/2pc-compose.cfg 0 0
     expose:
       - "7777"
     depends_on:
@@ -43,7 +43,7 @@ services:
     build: .
     image: opencbdc-tx
     tty: true
-    command: ./build/src/uhs/twophase/locking_shard/locking-shardd 2pc-compose.cfg 0 0
+    command: ./build/src/uhs/twophase/locking_shard/locking-shardd ./config/general/2pc-compose.cfg 0 0
     expose:
       - "6666"
     ports:

config.cfg.sample config/general/config.cfg.sample

@@ -6,7 +6,7 @@ services:
     build: .
     image: opencbdc-tx
     tty: true
-    command: ./build/src/uhs/atomizer/watchtower/watchtowerd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/watchtower/watchtowerd ./config/general/atomizer-compose.cfg 0
     ports:
       - 8555:8555
     expose:
@@ -23,7 +23,7 @@ services:
     build: .
     image: opencbdc-tx
     tty: true
-    command: ./build/src/uhs/atomizer/atomizer/atomizer-raftd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/atomizer/atomizer-raftd ./config/general/atomizer-compose.cfg 0
     expose:
       - "5555"
     depends_on:
@@ -40,7 +40,7 @@ services:
     build: .
     image: opencbdc-tx
     tty: true
-    command: ./build/src/uhs/atomizer/archiver/archiverd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/archiver/archiverd ./config/general/atomizer-compose.cfg 0
     expose:
       - "4555"
     depends_on:
@@ -59,7 +59,7 @@ services:
     build: .
     image: opencbdc-tx
     tty: true
-    command: ./build/src/uhs/atomizer/shard/shardd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/shard/shardd ./config/general/atomizer-compose.cfg 0
     expose:
       - "6555"
     depends_on:
@@ -79,7 +79,7 @@ services:
     build: .
     image: opencbdc-tx
     tty: true
-    command: ./build/src/uhs/atomizer/sentinel/sentineld atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/sentinel/sentineld ./config/general/atomizer-compose.cfg 0
     ports:
       - 7555:7555
     depends_on:

multimachine.cfg.sample config/general/multimachine.cfg.sample

@@ -6,6 +6,6 @@ services:
     build: .
     image: opencbdc-tx
     tty: true
-    command: sh ./scripts/test-transaction.sh 2pc-compose.cfg
+    command: sh ./scripts/test-transaction.sh ./config/general/2pc-compose.cfg
     networks:
       - 2pc-network

tests/integration/integration_tests.cfg config/integration/integration_tests.cfg

@@ -164,7 +164,7 @@ run_test_suite () {
 if [[ "$RUN_UNIT_TESTS" == "true" ]]
 then
     echo "Running unit tests..."
-    find "${REPO_TOP_DIR}"/tests/unit/ -name '*.cfg' \
+    find "${REPO_TOP_DIR}"/config/unit/ -name '*.cfg' \
         -exec rsync \{\} "$BUILD_DIR" \;
     run_test_suite "tests/unit/run_unit_tests" "unit_tests_coverage"
 else
@@ -175,7 +175,8 @@ echo
 if [[ "$RUN_INTEGRATION_TESTS" == "true" ]]
 then
     echo "Running integration tests..."
-    cp "${REPO_TOP_DIR}"/tests/integration/*.cfg "$BUILD_DIR"
+    cp "${REPO_TOP_DIR}"/config/integration/*.cfg "${BUILD_DIR}"
+    cp "${REPO_TOP_DIR}"/tools/config_generator/*.tmpl "${BUILD_DIR}"/tools/config_generator
     run_test_suite "tests/integration/run_integration_tests" \
         "integration_tests_coverage"
 else

tests/integration/integration_tests_2pc.cfg config/integration/integration_tests_2pc.cfg

@@ -1,6 +1,6 @@
 project(tests)
 
-include_directories(. ../src ../tools/watchtower ../3rdparty ../3rdparty/secp256k1/include)
+include_directories(. ../src ../ ../tools/config_generator ../tools/watchtower ../3rdparty ../3rdparty/secp256k1/include)
 set(SECP256K1_LIBRARY $<TARGET_FILE:secp256k1>)
 
 add_library(util util.cpp)

tests/integration/replicated_atomizer.cfg config/integration/replicated_atomizer.cfg

@@ -5,6 +5,7 @@ add_executable(run_unit_tests archiver_test.cpp
                               atomizer_test.cpp
                               buffer_test.cpp
                               common/hash_test.cpp
+                              config_gen_test.cpp
                               config_test.cpp
                               coordinator/messages_test.cpp
                               locking_shard/format_test.cpp
@@ -50,6 +51,7 @@ target_link_libraries(run_unit_tests ${GTEST_LIBRARY}
                                      transaction
                                      network
                                      common
+                                     config_generator
                                      serialization
                                      crypto
                                      secp256k1

tests/integration/replicated_shard.cfg config/integration/replicated_shard.cfg

@@ -0,0 +1,61 @@
+// Copyright (c) 2022 MIT Digital Currency Initiative,
+//                    Federal Reserve Bank of Boston
+//                    MITRE Corporation
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "tools/config_generator/config_generator.hpp"
+#include "util/common/config.hpp"
+
+#include <filesystem>
+#include <gtest/gtest.h>
+#include <string>
+
+// TODO: Add file parsing tests.
+
+class config_generation_validation_test : public ::testing::Test {
+  protected:
+    void SetUp() override {
+        template_file_atomizer
+            = "../config/unit/atomizer_template_unit_test.tmpl";
+        template_file_2pc = "../config/unit/2pc_template_unit_test.tmpl";
+        port_num = 5555;
+    }
+
+    std::string template_file_atomizer;
+    std::string template_file_2pc;
+    size_t port_num;
+};
+
+TEST_F(config_generation_validation_test,
+       generate_configuration_file_atomizer_test) {
+    // Assumes build dir is "build". Cannot find a way around this since unit
+    // tests can be run from either root dir or build dir and we don't
+    // necessarily know which one
+    std::string build_dir = "build";
+    cbdc::generate_config::config_generator new_config_gen(
+        template_file_atomizer,
+        port_num,
+        build_dir);
+    auto cfg_or_err = new_config_gen.generate_configuration_file();
+    ASSERT_EQ(cfg_or_err, "SUCCESS");
+    // TODO
+    // Reload generate file and check values
+    // Delete generated file
+}
+
+TEST_F(config_generation_validation_test,
+       generate_configuration_file_two_phase_test) {
+    // Assumes build dir is "build". Cannot find a way around this since unit
+    // tests can be run from either root dir or build dir and we don't
+    // necessarily know which one
+    std::string build_dir = "build";
+    cbdc::generate_config::config_generator new_config_gen(template_file_2pc,
+                                                           port_num,
+                                                           build_dir);
+    auto cfg_or_err = new_config_gen.generate_configuration_file();
+    ASSERT_EQ(cfg_or_err, "SUCCESS");
+    // TODO
+    // Reload generate file and check values
+    // Delete generated file
+}

config/unit/2pc_template_unit_test.tmpl

@@ -0,0 +1,44 @@
+2pc=1
+sentinel_count=1
+shard_count=1
+coordinator_count=1
+coordinator_max_threads=1
+election_timeout_upper=4000
+election_timeout_lower=3000
+heartbeat=1000
+raft_max_batch=100000
+snapshot_distance=1000000000
+batch_size=1
+wait_for_followers=0
+loadgen_invalid_tx_rate=0.00
+loadgen_fixed_tx_rate=0.01
+loadgen_sendtx_output_count=1
+loadgen_sendtx_input_count=1
+initial_mint_count=20000
+initial_mint_value=100
+tmpl_randomize_values=1
+tmpl_shard_start=0
+tmpl_shard_size=255
+tmpl_max_shard_raft_replication_count=1
+tmpl_avg_shard_start_end_overlap_percent=0.15
+tmpl_max_coordinator_raft_replication_count=1
+tmpl_default_log_level="INFO"
+tmpl_universal_override_log_level="WARN"
+tmpl_sentinel_log_level="WARN"
+tmpl_coordinator_log_level="DEBUG"
+tmpl_shard_log_level="DEBUG"
+num_wallets=10
+num_minters=1
+num_redeemers=1
+total_number_of_transactions=100
+avg_mint_value=10
+avg_mint_count=10
+avg_redemption_value=5
+avg_redemption_count=5
+transaction_frequency=5
+mint_frequency=0.1
+redemption_frequency=0.05
+sentinel_offline_probability=0.01
+shard_offline_probability=0.01
+coordinator_offline_probability=0.01
+randomize_execution=0