Completes PR #138, adds additional contemperary features

Removes identification of build directory, as sending file to
build directory has been replaced by priting config file to
stdout. Also adds support for seeding by generating a
seed key if necessary.

Update to properly print out config file to std error,
and merge conventions such that m_new_config is used
to store config file data

Signed-off-by: Michael Maurer <[email protected]>

Author: maurermi

CMakeLists.txt

@@ -76,5 +76,6 @@ endif()
 add_subdirectory(src)
 add_subdirectory(tests)
 add_subdirectory(benchmarks)
+add_subdirectory(tools/config_generator)
 add_subdirectory(tools/bench)
 add_subdirectory(tools/shard-seeder)

README.md

@@ -160,38 +160,38 @@ You can find the images [in the Github Container Registry](https://github.com/mi
 ## Setup test wallets and test them
 
 The following commands are all performed from within the second container we started in the previous step.
-In each of the below commands, you should pass `atomizer-compose.cfg` instead of `2pc-compose.cfg` if you started the atomizer architecture.
+In each of the below commands, you should pass `config/general/atomizer-compose.cfg` instead of `config/general/2pc-compose.cfg` if you started the atomizer architecture.
 
 * Mint new coins (e.g., 10 new UTXOs each with a value of 5 atomic units of currency)
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat mint 10 5
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat mint 10 5
   [2021-08-17 15:11:57.686] [WARN ] Existing wallet file not found
   [2021-08-17 15:11:57.686] [WARN ] Existing mempool not found
   4bc23da407c3a8110145c5b6c38199c8ec3b0e35ea66bbfd78f0ed65304ce6fa
   ```
 
   If using the atomizer architecture, you'll need to sync the wallet after:
   ```terminal
-  # ./build/src/uhs/client/client-cli atomizer-compose.cfg mempool0.dat wallet0.dat sync
+  # ./build/src/uhs/client/client-cli config/general/atomizer-compose.cfg mempool0.dat wallet0.dat sync
   ```
 
 * Inspect the balance of a wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat info
   Balance: $0.50, UTXOs: 10, pending TXs: 0
   ```
 
 * Make a new wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat newaddress
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat newaddress
   [2021-08-17 15:13:16.148] [WARN ] Existing wallet file not found
   [2021-08-17 15:13:16.148] [WARN ] Existing mempool not found
   usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
   ```
 
 * Send currency from the first wallet to the second wallet created in the previous step (e.g., 30 atomic units of currency)
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat send 30 usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat send 30 usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
   tx_id:
   cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e75
   Data for recipient importinput:
@@ -201,20 +201,20 @@ In each of the below commands, you should pass `atomizer-compose.cfg` instead of
 
   If using the atomizer architecture, you'll need to sync the sending wallet after:
   ```terminal
-  # ./build/src/uhs/client/client-cli atomizer-compose.cfg mempool0.dat wallet0.dat sync
+  # ./build/src/uhs/client/client-cli config/general/atomizer-compose.cfg mempool0.dat wallet0.dat sync
   ```
 
 * Check that the currency is no longer available in the sending wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat info
   Balance: $0.20, UTXOs: 4, pending TXs: 0
   ```
 
 * Import coins to the receiving wallet using the string after `importinput` from the currency transfer step above
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat importinput cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e750000000000000000d0e4f689b550f623e9370edae235de50417860be0f2f8e924eca9f402fcefeaa1e00000000000000
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat sync
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat importinput cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e750000000000000000d0e4f689b550f623e9370edae235de50417860be0f2f8e924eca9f402fcefeaa1e00000000000000
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat sync
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat info
   Balance: $0.30, UTXOs: 1, pending TXs: 0
   ```
 

2pc-compose.cfg config/general/2pc-compose.cfg

@@ -0,0 +1,44 @@
+2pc=1
+sentinel_count=1
+shard_count=1
+coordinator_count=1
+coordinator_max_threads=1
+election_timeout_upper=4000
+election_timeout_lower=3000
+heartbeat=1000
+raft_max_batch=100000
+snapshot_distance=1000000000
+batch_size=1
+wait_for_followers=0
+loadgen_invalid_tx_rate=0.00
+loadgen_fixed_tx_rate=0.01
+loadgen_sendtx_output_count=1
+loadgen_sendtx_input_count=1
+initial_mint_count=20000
+initial_mint_value=100
+tmpl_randomize_values=1
+tmpl_shard_start=0
+tmpl_shard_size=255
+tmpl_max_shard_raft_replication_count=1
+tmpl_avg_shard_start_end_overlap_percent=0.15
+tmpl_max_coordinator_raft_replication_count=1
+tmpl_default_log_level="INFO"
+tmpl_universal_override_log_level="WARN"
+tmpl_sentinel_log_level="WARN"
+tmpl_coordinator_log_level="DEBUG"
+tmpl_shard_log_level="DEBUG"
+num_wallets=10
+num_minters=1
+num_redeemers=1
+total_number_of_transactions=100
+avg_mint_value=10
+avg_mint_count=10
+avg_redemption_value=5
+avg_redemption_count=5
+transaction_frequency=5
+mint_frequency=0.1
+redemption_frequency=0.05
+sentinel_offline_probability=0.01
+shard_offline_probability=0.01
+coordinator_offline_probability=0.01
+randomize_execution=0

2pc.cfg.sample config/general/2pc.cfg.sample

@@ -0,0 +1,32 @@
+2pc=0
+archiver_count=1
+atomizer_count=1
+shard_count=1
+sentinel_count=1
+watchtower_count=1
+target_block_interval=3000
+stxo_cache_depth=2
+target_block_interval=250
+election_timeout_upper=4000
+election_timeout_lower=3000
+heartbeat=1000
+raft_max_batch=100000
+snapshot_distance=1000000000
+batch_size=1
+wait_for_followers=0
+loadgen_invalid_tx_rate=0.00
+loadgen_fixed_tx_rate=0.01
+loadgen_sendtx_output_count=1
+loadgen_sendtx_input_count=1
+initial_mint_count=20000
+initial_mint_value=100
+tmpl_randomize_values=1
+tmpl_shard_start=0
+tmpl_shard_size=255
+tmpl_avg_shard_start_end_overlap_percent=0.15
+tmpl_default_log_level="INFO"
+tmpl_sentinel_log_level="DEBUG"
+tmpl_shard_log_level="INFO"
+tmpl_watchtower_log_level="DEBUG"
+tmpl_archiver_log_level="DEBUG"
+tmpl_atomizer_log_level="DEBUG"

atomizer-compose.cfg config/general/atomizer-compose.cfg

@@ -9,7 +9,7 @@ services:
     image: opencbdc-tx-twophase
     tty: true
     restart: always
-    command: ./build/src/uhs/twophase/sentinel_2pc/sentineld-2pc 2pc-compose.cfg 0
+    command: ./build/src/uhs/twophase/sentinel_2pc/sentineld-2pc ./config/general/2pc-compose.cfg 0
     ports:
       - 5555:5555
     depends_on:
@@ -29,7 +29,7 @@ services:
       target: twophase
     image: opencbdc-tx-twophase
     tty: true
-    command: ./build/src/uhs/twophase/coordinator/coordinatord 2pc-compose.cfg 0 0
+    command: ./build/src/uhs/twophase/coordinator/coordinatord ./config/general/2pc-compose.cfg 0 0
     expose:
       - "7777"
     depends_on:
@@ -49,7 +49,7 @@ services:
       target: twophase
     image: opencbdc-tx-twophase
     tty: true
-    command: ./build/src/uhs/twophase/locking_shard/locking-shardd 2pc-compose.cfg 0 0
+    command: ./build/src/uhs/twophase/locking_shard/locking-shardd ./config/general/2pc-compose.cfg 0 0
     expose:
       - "6666"
     ports:

config.cfg.sample config/general/config.cfg.sample

@@ -8,7 +8,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/watchtower/watchtowerd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/watchtower/watchtowerd ./config/general/atomizer-compose.cfg 0
     ports:
       - 8555:8555
     expose:
@@ -28,7 +28,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/atomizer/atomizer-raftd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/atomizer/atomizer-raftd ./config/general/atomizer-compose.cfg 0
     expose:
       - "5555"
     depends_on:
@@ -48,7 +48,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/archiver/archiverd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/archiver/archiverd ./config/general/atomizer-compose.cfg 0
     expose:
       - "4555"
     depends_on:
@@ -69,7 +69,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/shard/shardd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/shard/shardd ./config/general/atomizer-compose.cfg 0
     expose:
       - "6555"
     depends_on:
@@ -91,7 +91,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/sentinel/sentineld atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/sentinel/sentineld ./config/general/atomizer-compose.cfg 0
     ports:
       - 7555:7555
     depends_on:

multimachine.cfg.sample config/general/multimachine.cfg.sample

@@ -164,7 +164,7 @@ run_test_suite () {
 if [[ "$RUN_UNIT_TESTS" == "true" ]]
 then
     echo "Running unit tests..."
-    find "${REPO_TOP_DIR}"/tests/unit/ -name '*.cfg' \
+    find "${REPO_TOP_DIR}"/config/unit/ -name '*.cfg' \
         -exec rsync \{\} "$BUILD_DIR" \;
     run_test_suite "tests/unit/run_unit_tests" "unit_tests_coverage"
 else
@@ -175,7 +175,8 @@ echo
 if [[ "$RUN_INTEGRATION_TESTS" == "true" ]]
 then
     echo "Running integration tests..."
-    cp "${REPO_TOP_DIR}"/tests/integration/*.cfg "$BUILD_DIR"
+    cp "${REPO_TOP_DIR}"/config/integration/*.cfg "${BUILD_DIR}"
+    cp "${REPO_TOP_DIR}"/tools/config_generator/*.tmpl "${BUILD_DIR}"/tools/config_generator
     run_test_suite "tests/integration/run_integration_tests" \
         "integration_tests_coverage"
 else

tests/integration/integration_tests.cfg config/integration/integration_tests.cfg

@@ -1,6 +1,6 @@
 project(tests)
 
-include_directories(. ../src ../tools/watchtower ../3rdparty ../3rdparty/secp256k1/include)
+include_directories(. ../src ../ ../tools/config_generator ../tools/watchtower ../3rdparty ../3rdparty/secp256k1/include)
 set(SECP256K1_LIBRARY $<TARGET_FILE:secp256k1>)
 
 add_library(util util.cpp)

tests/integration/integration_tests_2pc.cfg config/integration/integration_tests_2pc.cfg

@@ -5,6 +5,7 @@ add_executable(run_unit_tests archiver_test.cpp
                               atomizer_test.cpp
                               buffer_test.cpp
                               common/hash_test.cpp
+                              config_gen_test.cpp
                               config_test.cpp
                               coordinator/messages_test.cpp
                               locking_shard/format_test.cpp
@@ -50,6 +51,7 @@ target_link_libraries(run_unit_tests ${GTEST_LIBRARY}
                                      transaction
                                      network
                                      common
+                                     config_generator
                                      serialization
                                      crypto
                                      secp256k1

tests/integration/replicated_atomizer.cfg config/integration/replicated_atomizer.cfg

@@ -0,0 +1,57 @@
+// Copyright (c) 2022 MIT Digital Currency Initiative,
+//                    Federal Reserve Bank of Boston
+//                    MITRE Corporation
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "tools/config_generator/config_generator.hpp"
+#include "util/common/config.hpp"
+
+#include <filesystem>
+#include <gtest/gtest.h>
+#include <string>
+
+// TODO: Add file parsing tests.
+
+class config_generation_validation_test : public ::testing::Test {
+  protected:
+    void SetUp() override {
+        template_file_atomizer
+            = "../config/unit/atomizer_template_unit_test.tmpl";
+        template_file_2pc = "../config/unit/2pc_template_unit_test.tmpl";
+        port_num = 5555;
+    }
+
+    std::string template_file_atomizer;
+    std::string template_file_2pc;
+    size_t port_num;
+};
+
+TEST_F(config_generation_validation_test,
+       generate_configuration_file_atomizer_test) {
+    // Assumes build dir is "build". Cannot find a way around this since unit
+    // tests can be run from either root dir or build dir and we don't
+    // necessarily know which one
+    cbdc::generate_config::config_generator new_config_gen(
+        template_file_atomizer,
+        port_num);
+    auto cfg_or_err = new_config_gen.generate_configuration_file();
+    ASSERT_EQ(cfg_or_err, "SUCCESS");
+    // TODO
+    // Reload generate file and check values
+    // Delete generated file
+}
+
+TEST_F(config_generation_validation_test,
+       generate_configuration_file_two_phase_test) {
+    // Assumes build dir is "build". Cannot find a way around this since unit
+    // tests can be run from either root dir or build dir and we don't
+    // necessarily know which one
+    cbdc::generate_config::config_generator new_config_gen(template_file_2pc,
+                                                           port_num);
+    auto cfg_or_err = new_config_gen.generate_configuration_file();
+    ASSERT_EQ(cfg_or_err, "SUCCESS");
+    // TODO
+    // Reload generate file and check values
+    // Delete generated file
+}

tests/integration/replicated_shard.cfg config/integration/replicated_shard.cfg

@@ -0,0 +1,44 @@
+2pc=1
+sentinel_count=1
+shard_count=1
+coordinator_count=1
+coordinator_max_threads=1
+election_timeout_upper=4000
+election_timeout_lower=3000
+heartbeat=1000
+raft_max_batch=100000
+snapshot_distance=1000000000
+batch_size=1
+wait_for_followers=0
+loadgen_invalid_tx_rate=0.00
+loadgen_fixed_tx_rate=0.01
+loadgen_sendtx_output_count=1
+loadgen_sendtx_input_count=1
+initial_mint_count=20000
+initial_mint_value=100
+tmpl_randomize_values=1
+tmpl_shard_start=0
+tmpl_shard_size=255
+tmpl_max_shard_raft_replication_count=1
+tmpl_avg_shard_start_end_overlap_percent=0.15
+tmpl_max_coordinator_raft_replication_count=1
+tmpl_default_log_level="INFO"
+tmpl_universal_override_log_level="WARN"
+tmpl_sentinel_log_level="WARN"
+tmpl_coordinator_log_level="DEBUG"
+tmpl_shard_log_level="DEBUG"
+num_wallets=10
+num_minters=1
+num_redeemers=1
+total_number_of_transactions=100
+avg_mint_value=10
+avg_mint_count=10
+avg_redemption_value=5
+avg_redemption_count=5
+transaction_frequency=5
+mint_frequency=0.1
+redemption_frequency=0.05
+sentinel_offline_probability=0.01
+shard_offline_probability=0.01
+coordinator_offline_probability=0.01
+randomize_execution=0