Completes PR mit-dci#138, adds additional contemperary features

Signed-off-by: Michael Maurer <[email protected]>

Author: maurermi

2pc-compose.cfg

@@ -18,3 +18,4 @@ shard0_loglevel="INFO"
 shard0_0_endpoint="shard0:6666"
 shard0_0_raft_endpoint="shard0:6667"
 shard0_0_readonly_endpoint="shard0:6767"
+

CMakeLists.txt

@@ -76,5 +76,6 @@ endif()
 add_subdirectory(src)
 add_subdirectory(tests)
 add_subdirectory(benchmarks)
+add_subdirectory(tools/config_generator)
 add_subdirectory(tools/bench)
 add_subdirectory(tools/shard-seeder)

README.md

@@ -160,38 +160,38 @@ You can find the images [in the Github Container Registry](https://github.com/mi
 ## Setup test wallets and test them
 
 The following commands are all performed from within the second container we started in the previous step.
-In each of the below commands, you should pass `atomizer-compose.cfg` instead of `2pc-compose.cfg` if you started the atomizer architecture.
+In each of the below commands, you should pass `config/general/atomizer-compose.cfg` instead of `config/general/2pc-compose.cfg` if you started the atomizer architecture.
 
 * Mint new coins (e.g., 10 new UTXOs each with a value of 5 atomic units of currency)
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat mint 10 5
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat mint 10 5
   [2021-08-17 15:11:57.686] [WARN ] Existing wallet file not found
   [2021-08-17 15:11:57.686] [WARN ] Existing mempool not found
   4bc23da407c3a8110145c5b6c38199c8ec3b0e35ea66bbfd78f0ed65304ce6fa
   ```
 
   If using the atomizer architecture, you'll need to sync the wallet after:
   ```terminal
-  # ./build/src/uhs/client/client-cli atomizer-compose.cfg mempool0.dat wallet0.dat sync
+  # ./build/src/uhs/client/client-cli config/general/atomizer-compose.cfg mempool0.dat wallet0.dat sync
   ```
 
 * Inspect the balance of a wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat info
   Balance: $0.50, UTXOs: 10, pending TXs: 0
   ```
 
 * Make a new wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat newaddress
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat newaddress
   [2021-08-17 15:13:16.148] [WARN ] Existing wallet file not found
   [2021-08-17 15:13:16.148] [WARN ] Existing mempool not found
   usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
   ```
 
 * Send currency from the first wallet to the second wallet created in the previous step (e.g., 30 atomic units of currency)
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat send 30 usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat send 30 usd1qrw038lx5n4wxx3yvuwdndpr7gnm347d6pn37uywgudzq90w7fsuk52kd5u
   tx_id:
   cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e75
   Data for recipient importinput:
@@ -201,20 +201,20 @@ In each of the below commands, you should pass `atomizer-compose.cfg` instead of
 
   If using the atomizer architecture, you'll need to sync the sending wallet after:
   ```terminal
-  # ./build/src/uhs/client/client-cli atomizer-compose.cfg mempool0.dat wallet0.dat sync
+  # ./build/src/uhs/client/client-cli config/general/atomizer-compose.cfg mempool0.dat wallet0.dat sync
   ```
 
 * Check that the currency is no longer available in the sending wallet
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool0.dat wallet0.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool0.dat wallet0.dat info
   Balance: $0.20, UTXOs: 4, pending TXs: 0
   ```
 
 * Import coins to the receiving wallet using the string after `importinput` from the currency transfer step above
   ```terminal
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat importinput cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e750000000000000000d0e4f689b550f623e9370edae235de50417860be0f2f8e924eca9f402fcefeaa1e00000000000000
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat sync
-  # ./build/src/uhs/client/client-cli 2pc-compose.cfg mempool1.dat wallet1.dat info
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat importinput cc1f7dc708be5b07e23e125cf0674002ff8546a9342928114bc97031d8b96e750000000000000000d0e4f689b550f623e9370edae235de50417860be0f2f8e924eca9f402fcefeaa1e00000000000000
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat sync
+  # ./build/src/uhs/client/client-cli config/general/2pc-compose.cfg mempool1.dat wallet1.dat info
   Balance: $0.30, UTXOs: 1, pending TXs: 0
   ```
 

atomizer-compose.cfg

@@ -1,21 +1,40 @@
-atomizer_count=1
-atomizer0_endpoint="atomizer0:5555"
-atomizer0_raft_endpoint="atomizer0:6666"
+2pc=0
 archiver_count=1
-archiver0_endpoint="archiver0:4555"
-archiver0_db="archiver0_db"
+atomizer_count=1
+batch_size=1
+election_timeout_lower=3000
+election_timeout_upper=4000
+heartbeat=1000
+initial_mint_count=20000
+initial_mint_value=100
+loadgen_fixed_tx_rate=0.01
+loadgen_invalid_tx_rate=0
+loadgen_sendtx_input_count=1
+loadgen_sendtx_output_count=1
+raft_max_batch=100000
+sentinel_count=1
 shard_count=1
-shard0_endpoint="shard0:6555"
+snapshot_distance=1000000000
+stxo_cache_depth=2
+target_block_interval=3000
+wait_for_followers=0
+watchtower_count=1
+shard0_endpoint="127.0.0.1:1024"
+shard0_db="shard0_db"
+shard0_loglevel="INFO"
 shard0_start=0
 shard0_end=255
-shard0_db="shard0"
-sentinel_count=1
-sentinel0_endpoint="sentinel0:7555"
+sentinel0_endpoint="127.0.0.1:1025"
 sentinel0_loglevel="INFO"
-sentinel0_private_key="0000000000000001000000000000000000000000000000000000000000000000"
-sentinel0_public_key="eaa649f21f51bdbae7be4ae34ce6e5217a58fdce7f47f9aa7f3b58fa2120e2b3"
-watchtower_count=1
-watchtower0_client_endpoint="watchtower0:8555"
-watchtower0_internal_endpoint="watchtower0:8556"
-watchtower_block_cache_size=50
-target_block_interval=3000
+sentinel0_private_key="61e5bdb0d8469c23ae85c2dd0e36351ada18bc03ea32bc6e450bda23ad6d7f01"
+sentinel0_endpoint="092614a67554f4a9437d4ce81749f9ea18f46dd50533c9e7820e5773822c0125"
+archiver0_endpoint="127.0.0.1:1026"
+archiver0_db="archiver0_db"
+archiver0_loglevel="INFO"
+atomizer0_endpoint="127.0.0.1:1027"
+atomizer0_raft_endpoint="127.0.0.1:1028"
+atomizer0_loglevel="INFO"
+watchtower0_client_endpoint="127.0.0.1:1029"
+watchtower0_internal_endpoint="127.0.0.1:1030"
+watchtower0_loglevel="INFO"
+

config/general/2pc-compose.cfg

@@ -0,0 +1,20 @@
+2pc=1
+sentinel_count=1
+sentinel0_endpoint="sentinel0:5555"
+sentinel0_loglevel="WARN"
+sentinel0_private_key="0000000000000001000000000000000000000000000000000000000000000000"
+sentinel0_public_key="eaa649f21f51bdbae7be4ae34ce6e5217a58fdce7f47f9aa7f3b58fa2120e2b3"
+coordinator_count=1
+coordinator0_count=1
+coordinator0_loglevel="INFO"
+coordinator0_0_endpoint="coordinator0:7777"
+coordinator0_0_raft_endpoint="coordinator0:7778"
+coordinator_max_threads=100
+shard_count=1
+shard0_start=0
+shard0_end=255
+shard0_count=1
+shard0_loglevel="INFO"
+shard0_0_endpoint="shard0:6666"
+shard0_0_raft_endpoint="shard0:6667"
+shard0_0_readonly_endpoint="shard0:6767"

2pc.cfg.sample config/general/2pc.cfg.sample

@@ -0,0 +1,21 @@
+atomizer_count=1
+atomizer0_endpoint="atomizer0:5555"
+atomizer0_raft_endpoint="atomizer0:6666"
+archiver_count=1
+archiver0_endpoint="archiver0:4555"
+archiver0_db="archiver0_db"
+shard_count=1
+shard0_endpoint="shard0:6555"
+shard0_start=0
+shard0_end=255
+shard0_db="shard0"
+sentinel_count=1
+sentinel0_endpoint="sentinel0:7555"
+sentinel0_loglevel="INFO"
+sentinel0_private_key="0000000000000001000000000000000000000000000000000000000000000000"
+sentinel0_public_key="eaa649f21f51bdbae7be4ae34ce6e5217a58fdce7f47f9aa7f3b58fa2120e2b3"
+watchtower_count=1
+watchtower0_client_endpoint="watchtower0:8555"
+watchtower0_internal_endpoint="watchtower0:8556"
+watchtower_block_cache_size=50
+target_block_interval=3000

config/general/atomizer-compose.cfg

@@ -0,0 +1,44 @@
+2pc=1
+sentinel_count=1
+shard_count=1
+coordinator_count=1
+coordinator_max_threads=1
+election_timeout_upper=4000
+election_timeout_lower=3000
+heartbeat=1000
+raft_max_batch=100000
+snapshot_distance=1000000000
+batch_size=1
+wait_for_followers=0
+loadgen_invalid_tx_rate=0.00
+loadgen_fixed_tx_rate=0.01
+loadgen_sendtx_output_count=1
+loadgen_sendtx_input_count=1
+initial_mint_count=20000
+initial_mint_value=100
+tmpl_randomize_values=1
+tmpl_shard_start=0
+tmpl_shard_size=255
+tmpl_max_shard_raft_replication_count=1
+tmpl_avg_shard_start_end_overlap_percent=0.15
+tmpl_max_coordinator_raft_replication_count=1
+tmpl_default_log_level="INFO"
+tmpl_universal_override_log_level="WARN"
+tmpl_sentinel_log_level="WARN"
+tmpl_coordinator_log_level="DEBUG"
+tmpl_shard_log_level="DEBUG"
+num_wallets=10
+num_minters=1
+num_redeemers=1
+total_number_of_transactions=100
+avg_mint_value=10
+avg_mint_count=10
+avg_redemption_value=5
+avg_redemption_count=5
+transaction_frequency=5
+mint_frequency=0.1
+redemption_frequency=0.05
+sentinel_offline_probability=0.01
+shard_offline_probability=0.01
+coordinator_offline_probability=0.01
+randomize_execution=0

config.cfg.sample config/general/config.cfg.sample

@@ -0,0 +1,32 @@
+2pc=0
+archiver_count=1
+atomizer_count=1
+shard_count=1
+sentinel_count=1
+watchtower_count=1
+target_block_interval=3000
+stxo_cache_depth=2
+target_block_interval=250
+election_timeout_upper=4000
+election_timeout_lower=3000
+heartbeat=1000
+raft_max_batch=100000
+snapshot_distance=1000000000
+batch_size=1
+wait_for_followers=0
+loadgen_invalid_tx_rate=0.00
+loadgen_fixed_tx_rate=0.01
+loadgen_sendtx_output_count=1
+loadgen_sendtx_input_count=1
+initial_mint_count=20000
+initial_mint_value=100
+tmpl_randomize_values=1
+tmpl_shard_start=0
+tmpl_shard_size=255
+tmpl_avg_shard_start_end_overlap_percent=0.15
+tmpl_default_log_level="INFO"
+tmpl_sentinel_log_level="DEBUG"
+tmpl_shard_log_level="INFO"
+tmpl_watchtower_log_level="DEBUG"
+tmpl_archiver_log_level="DEBUG"
+tmpl_atomizer_log_level="DEBUG"

multimachine.cfg.sample config/general/multimachine.cfg.sample

@@ -9,7 +9,7 @@ services:
     image: opencbdc-tx-twophase
     tty: true
     restart: always
-    command: ./build/src/uhs/twophase/sentinel_2pc/sentineld-2pc 2pc-compose.cfg 0
+    command: ./build/src/uhs/twophase/sentinel_2pc/sentineld-2pc ./config/general/2pc-compose.cfg 0
     ports:
       - 5555:5555
     depends_on:
@@ -29,7 +29,7 @@ services:
       target: twophase
     image: opencbdc-tx-twophase
     tty: true
-    command: ./build/src/uhs/twophase/coordinator/coordinatord 2pc-compose.cfg 0 0
+    command: ./build/src/uhs/twophase/coordinator/coordinatord ./config/general/2pc-compose.cfg 0 0
     expose:
       - "7777"
     depends_on:
@@ -49,7 +49,7 @@ services:
       target: twophase
     image: opencbdc-tx-twophase
     tty: true
-    command: ./build/src/uhs/twophase/locking_shard/locking-shardd 2pc-compose.cfg 0 0
+    command: ./build/src/uhs/twophase/locking_shard/locking-shardd ./config/general/2pc-compose.cfg 0 0
     expose:
       - "6666"
     ports:

tests/integration/integration_tests.cfg config/integration/integration_tests.cfg

@@ -8,7 +8,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/watchtower/watchtowerd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/watchtower/watchtowerd ./config/general/atomizer-compose.cfg 0
     ports:
       - 8555:8555
     expose:
@@ -28,7 +28,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/atomizer/atomizer-raftd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/atomizer/atomizer-raftd ./config/general/atomizer-compose.cfg 0
     expose:
       - "5555"
     depends_on:
@@ -48,7 +48,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/archiver/archiverd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/archiver/archiverd ./config/general/atomizer-compose.cfg 0
     expose:
       - "4555"
     depends_on:
@@ -69,7 +69,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/shard/shardd atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/shard/shardd ./config/general/atomizer-compose.cfg 0
     expose:
       - "6555"
     depends_on:
@@ -91,7 +91,7 @@ services:
       target: atomizer
     image: opencbdc-tx-atomizer
     tty: true
-    command: ./build/src/uhs/atomizer/sentinel/sentineld atomizer-compose.cfg 0
+    command: ./build/src/uhs/atomizer/sentinel/sentineld ./config/general/atomizer-compose.cfg 0
     ports:
       - 7555:7555
     depends_on:

tests/integration/integration_tests_2pc.cfg config/integration/integration_tests_2pc.cfg

@@ -164,7 +164,7 @@ run_test_suite () {
 if [[ "$RUN_UNIT_TESTS" == "true" ]]
 then
     echo "Running unit tests..."
-    find "${REPO_TOP_DIR}"/tests/unit/ -name '*.cfg' \
+    find "${REPO_TOP_DIR}"/config/unit/ -name '*.cfg' \
         -exec rsync \{\} "$BUILD_DIR" \;
     run_test_suite "tests/unit/run_unit_tests" "unit_tests_coverage"
 else
@@ -175,7 +175,8 @@ echo
 if [[ "$RUN_INTEGRATION_TESTS" == "true" ]]
 then
     echo "Running integration tests..."
-    cp "${REPO_TOP_DIR}"/tests/integration/*.cfg "$BUILD_DIR"
+    cp "${REPO_TOP_DIR}"/config/integration/*.cfg "${BUILD_DIR}"
+    cp "${REPO_TOP_DIR}"/tools/config_generator/*.tmpl "${BUILD_DIR}"/tools/config_generator
     run_test_suite "tests/integration/run_integration_tests" \
         "integration_tests_coverage"
 else

tests/integration/replicated_atomizer.cfg config/integration/replicated_atomizer.cfg

@@ -1,6 +1,6 @@
 project(tests)
 
-include_directories(. ../src ../tools/watchtower ../3rdparty ../3rdparty/secp256k1/include)
+include_directories(. ../src ../ ../tools/config_generator ../tools/watchtower ../3rdparty ../3rdparty/secp256k1/include)
 set(SECP256K1_LIBRARY $<TARGET_FILE:secp256k1>)
 
 add_library(util util.cpp)

tests/integration/replicated_shard.cfg config/integration/replicated_shard.cfg

@@ -5,6 +5,7 @@ add_executable(run_unit_tests archiver_test.cpp
                               atomizer_test.cpp
                               buffer_test.cpp
                               common/hash_test.cpp
+                              config_gen_test.cpp
                               config_test.cpp
                               coordinator/messages_test.cpp
                               locking_shard/format_test.cpp
@@ -50,6 +51,7 @@ target_link_libraries(run_unit_tests ${GTEST_LIBRARY}
                                      transaction
                                      network
                                      common
+                                     config_generator
                                      serialization
                                      crypto
                                      secp256k1