From 3abee17a2f5384e7b08ba7ec446070e9124f61f4 Mon Sep 17 00:00:00 2001 From: GTC6244 <95836911+GTC6244@users.noreply.github.com> Date: Tue, 25 Nov 2025 21:17:21 -0500 Subject: [PATCH 1/5] Code compatibility - ie, it builds --- rust/lit-node/lit-node/src/git_info.rs | 2 +- rust/lit-node/lit-node/tests/test.rs | 2 +- .../tests/upgrades/version_upgrades.rs | 165 +++++------------- 3 files changed, 44 insertions(+), 125 deletions(-) diff --git a/rust/lit-node/lit-node/src/git_info.rs b/rust/lit-node/lit-node/src/git_info.rs index e7d0e87..ceeeca7 100644 --- a/rust/lit-node/lit-node/src/git_info.rs +++ b/rust/lit-node/lit-node/src/git_info.rs @@ -1 +1 @@ -pub const GIT_COMMIT_HASH: &str = "5caf4cb4b70f0ec3b5094e71785ce78421027805"; +pub const GIT_COMMIT_HASH: &str = "bc3891668f8441a1217a20343088e5067aa42188"; diff --git a/rust/lit-node/lit-node/tests/test.rs b/rust/lit-node/lit-node/tests/test.rs index 74505d8..9dfcc19 100644 --- a/rust/lit-node/lit-node/tests/test.rs +++ b/rust/lit-node/lit-node/tests/test.rs @@ -14,7 +14,7 @@ pub mod integration; // sdk tests - downloads the latest SDK & test it against the nodes in full compilation in a local network configuration pub mod sdk; // upgrade tests - test the upgrade process -//pub mod upgrades; +pub mod upgrades; // fault tests - test the fault tolerance of the network pub mod toxiproxy; diff --git a/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs b/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs index afc08e6..7af3c05 100644 --- a/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs +++ b/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs @@ -1,19 +1,17 @@ use lit_node_testnet::{ + TestSetupBuilder, node_collection::get_node_versions, testnet::{ NodeAccount, Testnet, - contracts::StakingContractRealmConfig, contracts_repo::{ self, WalletManifestItem, alias_node_configs_path, get_alias_manifest_template, latest_wallet_manifest, save_alias_manifest, }, }, - validator::ValidatorCollection, }; use crate::common::{ assertions::NetworkIntegrityChecker, - get_default_keyset_configs, init_test_config, version::{get_crate_version, update_node_crate_version}, }; @@ -29,52 +27,24 @@ use std::{fs, time::Duration}; use test_case::test_case; use tracing::info; -fn setup() { - setup_logging(); -} - /// Tests when an inactive validator that comes online with an invalid version, and then the staker requests to join, /// that the node should eventually be kicked for non-participation. #[tokio::test] async fn node_boot_invalid_version() { - setup(); - + crate::common::setup_logging(); info!("TEST: node_boot_invalid_version"); - // Set up a network with 6 nodes. let num_nodes = 6; // set epoch length to 30 mins so it never elapses unless we advance the clock let epoch_length = 1800; - let mut testnet = Testnet::builder() + + let (testnet, mut validator_collection, end_user) = TestSetupBuilder::default() .num_staked_and_joined_validators(num_nodes) - .num_staked_only_validators(1) .build() .await; - let testnet_contracts = Testnet::setup_contracts( - &mut testnet, - None, - Some( - StakingContractRealmConfig::builder() - .epoch_length(U256::from(epoch_length)) - .max_presign_count(U256::from(0)) - .min_presign_count(U256::from(0)) - .build(), - ), - ) - .await - .expect("Failed to setup contracts"); - - let actions = testnet.actions(testnet_contracts.contracts()); - - let mut validator_collection = ValidatorCollection::builder() - .num_staked_nodes(num_nodes) - .keyset_configs(get_default_keyset_configs()) - .build(&testnet, &actions) - .await - .expect("Failed to build validator collection"); - - let network_checker = NetworkIntegrityChecker::new(&actions).await; + let actions = testnet.actions(); + let network_checker = NetworkIntegrityChecker::new(&end_user, &actions).await; // Upgrade the node crate to a new version let _crate_version_handle = update_node_crate_version("2.9999.9999".to_string()); @@ -107,7 +77,7 @@ async fn node_boot_invalid_version() { let validator_to_kick = validator_collection .add_one( false, - Some(lit_node_testnet::validator::BuildMode::UseNewBuild), + Some(lit_node_testnet::validator::BuildMode::UseNewOrCachedBuild), None, ) .await @@ -121,14 +91,10 @@ async fn node_boot_invalid_version() { ); actions.increase_blockchain_timestamp(epoch_length).await; - let epoch_number = - actions - .get_current_epoch(realm_id) - .await; - + let epoch_number = actions.get_current_epoch(realm_id).await; + // Wait for kick - let voting_status = - actions + let voting_status = actions .wait_for_voting_status_to_kick_validator( realm_id, epoch_number, @@ -150,51 +116,27 @@ async fn node_boot_invalid_version() { actions.get_current_validator_count(realm_id).await as usize, num_nodes ); - network_checker.check(&validator_collection).await; + network_checker.check(&validator_collection, &vec![]).await; } /// Tests the version requirement change such that an active validator is running a node version that is incompatible, /// so it should request to leave. #[tokio::test] async fn active_validator_invalid_version() { - setup(); - + crate::common::setup_logging(); info!("TEST: active_validator_invalid_version"); - // Set up a network with 6 nodes. let num_nodes = 6; // set epoch length to 30 mins so it never elapses unless we advance the clock let epoch_length = 1800; - let mut testnet = Testnet::builder() + + let (testnet, mut validator_collection, end_user) = TestSetupBuilder::default() .num_staked_and_joined_validators(num_nodes) - .num_staked_only_validators(1) .build() .await; - let testnet_contracts = Testnet::setup_contracts( - &mut testnet, - None, - Some( - StakingContractRealmConfig::builder() - .epoch_length(U256::from(epoch_length)) - .max_presign_count(U256::from(0)) - .min_presign_count(U256::from(0)) - .build(), - ), - ) - .await - .expect("Failed to setup contracts"); - - let actions = testnet.actions(testnet_contracts.contracts()); - - let mut validator_collection = ValidatorCollection::builder() - .num_staked_nodes(num_nodes) - .keyset_configs(get_default_keyset_configs()) - .build(&testnet, &actions) - .await - .expect("Failed to build validator collection"); - - let network_checker = NetworkIntegrityChecker::new(&actions).await; + let actions = testnet.actions(); + let network_checker = NetworkIntegrityChecker::new(&end_user, &actions).await; // Upgrade the node crate to a new version let _crate_version_handle = update_node_crate_version("2.9999.9999".to_string()); @@ -204,7 +146,7 @@ async fn active_validator_invalid_version() { let new_validator = validator_collection .add_one( false, - Some(lit_node_testnet::validator::BuildMode::UseNewBuild), + Some(lit_node_testnet::validator::BuildMode::UseNewOrCachedBuild), None, ) .await @@ -228,7 +170,7 @@ async fn active_validator_invalid_version() { actions.get_current_validator_count(realm_id).await as usize, num_nodes + 1 ); - network_checker.check(&validator_collection).await; + network_checker.check(&validator_collection, &vec![]).await; // Update version requirements by setting a max version requirement, rendering the new node version invalid. let max_version = "2.9999.9998"; @@ -254,7 +196,7 @@ async fn active_validator_invalid_version() { actions.get_current_validator_count(realm_id).await as usize, num_nodes ); - network_checker.check(&validator_collection).await; + network_checker.check(&validator_collection, &vec![]).await; // Check that the new node is no longer a validator. let active_validators = actions.get_current_validators(realm_id).await; @@ -266,12 +208,10 @@ async fn active_validator_invalid_version() { /// 1. Run the `build_target_branches` script in the `scripts` directory. (x86 and arm64 builds) /// 2. Run the `download_builds` script in the `scripts` directory. (x86 builds only) /// The test will fail if the builds are not found. -#[test_case("origin/release-habanero-*"; "Upgrade against the latest Habanero release branch")] -#[test_case("origin/release-manzano-*"; "Upgrade against the latest Manzano release branch")] -#[test_case("origin/release-cayenne-*"; "Upgrade against the latest Cayenne release branch")] +#[test_case("origin/release-naga-prod-2025-11-25"; "Upgrade against the latest NAGA-Prod release branch")] #[tokio::test] async fn test_version_upgrade_against_old_version(target_branch: &str) { - setup(); + crate::common::setup_logging(); info!( "TEST: test_version_upgrade_against_old_version against {}", @@ -282,8 +222,10 @@ async fn test_version_upgrade_against_old_version(target_branch: &str) { let old_build_commit_hash = utils::get_target_branch_commit_hash(target_branch).expect("Failed to get commit hash"); + + info!("Old build commit hash: {}", old_build_commit_hash); // First check if we have the build. - let old_build_path = format!("./target/debug/lit_node_{}", old_build_commit_hash); + let old_build_path = format!("./target/test-run/debug/lit_node_{}", old_build_commit_hash); assert!( fs::metadata(&old_build_path).is_ok(), "Build does not exist at {}", @@ -292,49 +234,22 @@ async fn test_version_upgrade_against_old_version(target_branch: &str) { // Set up a network of nodes running the old build. + info!("TEST: node_boot_invalid_version"); + // Set up a network with 6 nodes. + let num_nodes = 6; // set epoch length to 30 mins so it never elapses unless we advance the clock let epoch_length = 1800; - // Start a new node collection and wait for the DKG to complete - // and root keys to be voted for. - let num_nodes = 5; - let mut testnet = Testnet::builder() + let (testnet, mut validator_collection, end_user) = TestSetupBuilder::default() .num_staked_and_joined_validators(num_nodes) .force_deploy(true) .build() .await; - let testnet_contracts = Testnet::setup_contracts( - &mut testnet, - None, - Some( - StakingContractRealmConfig::builder() - .epoch_length(U256::from(epoch_length)) - .min_presign_count(U256::from(0)) - .max_presign_count(U256::from(0)) - .max_presign_concurrency(U256::from(0)) - .realm_id(U256::from(1)) - .build(), - ), - ) - .await - .expect("Failed to setup contracts"); - - let actions = testnet.actions(testnet_contracts.contracts()); - - let mut validator_collection = ValidatorCollection::builder() - .num_staked_nodes(num_nodes) - .custom_binary_path(Some(old_build_path)) - .keyset_configs(get_default_keyset_configs()) - .build(&testnet, &actions) - .await - .expect("Failed to build validator collection"); - + let actions = testnet.actions(); let realm_id = U256::from(1); - let starting_epoch = validator_collection - .actions() - .get_current_epoch(realm_id) - .await; + let starting_epoch = actions.get_current_epoch(realm_id).await; + let mut next_epoch = starting_epoch + 1; // Keep track of the node versions. @@ -349,10 +264,11 @@ async fn test_version_upgrade_against_old_version(target_branch: &str) { .all(|v| v == &initial_node_versions[0]) ); - let network_checker = NetworkIntegrityChecker::new(validator_collection.actions()).await; - network_checker - .check_with_drained_presigns(&validator_collection) - .await; + let network_checker = NetworkIntegrityChecker::new(&end_user, &actions).await; + network_checker.check(&validator_collection, &vec![]).await; + // network_checker + // .check_with_drained_presigns(&validator_collection) + // .await; // First, we shuffle the order of the original staker wallets that we will be gradually adding aliases for. let mut wallet_manifest_wallets = latest_wallet_manifest(false); @@ -377,7 +293,7 @@ async fn test_version_upgrade_against_old_version(target_branch: &str) { false, alias_node_config_path, &get_latest_alias_node_account(0, &testnet), - Some(lit_node_testnet::validator::BuildMode::UseNewBuild), + Some(lit_node_testnet::validator::BuildMode::UseNewOrCachedBuild), 1 ) .await @@ -478,10 +394,13 @@ async fn test_version_upgrade_against_old_version(target_branch: &str) { fn get_latest_alias_node_account(idx: usize, testnet: &Testnet) -> NodeAccount { let latest_alias_wallet_manifest = latest_wallet_manifest(true); - let mut provider = ENDPOINT_MANAGER + let provider = ENDPOINT_MANAGER .get_provider(testnet.chain_name.clone()) .expect("Failed to get provider"); - provider.set_interval(Duration::new(0, 10)); + + let mut provider_mut = provider.as_ref().clone(); + provider_mut.set_interval(Duration::new(0, 10)); + let provider = std::sync::Arc::new(provider_mut); latest_alias_wallet_manifest[idx].map_to_node_account(provider, testnet.chain_id) } From 385303e8918a840ea95eaba7719a624a812f2883 Mon Sep 17 00:00:00 2001 From: GTC6244 <95836911+GTC6244@users.noreply.github.com> Date: Tue, 25 Nov 2025 21:19:42 -0500 Subject: [PATCH 2/5] cargo fmt --- rust/lit-node/lit-node/src/git_info.rs | 2 +- rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/lit-node/lit-node/src/git_info.rs b/rust/lit-node/lit-node/src/git_info.rs index ceeeca7..9bcec2e 100644 --- a/rust/lit-node/lit-node/src/git_info.rs +++ b/rust/lit-node/lit-node/src/git_info.rs @@ -1 +1 @@ -pub const GIT_COMMIT_HASH: &str = "bc3891668f8441a1217a20343088e5067aa42188"; +pub const GIT_COMMIT_HASH: &str = "3abee17a2f5384e7b08ba7ec446070e9124f61f4"; diff --git a/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs b/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs index 7af3c05..31770f6 100644 --- a/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs +++ b/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs @@ -222,7 +222,6 @@ async fn test_version_upgrade_against_old_version(target_branch: &str) { let old_build_commit_hash = utils::get_target_branch_commit_hash(target_branch).expect("Failed to get commit hash"); - info!("Old build commit hash: {}", old_build_commit_hash); // First check if we have the build. let old_build_path = format!("./target/test-run/debug/lit_node_{}", old_build_commit_hash); From c64d897e2975dbd579788c551935acd4bc497ef4 Mon Sep 17 00:00:00 2001 From: Brendon <95836911+GTC6244@users.noreply.github.com> Date: Tue, 25 Nov 2025 22:11:44 -0500 Subject: [PATCH 3/5] Update rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs b/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs index 31770f6..5f9cab4 100644 --- a/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs +++ b/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs @@ -233,7 +233,7 @@ async fn test_version_upgrade_against_old_version(target_branch: &str) { // Set up a network of nodes running the old build. - info!("TEST: node_boot_invalid_version"); + info!("TEST: test_version_upgrade_against_old_version"); // Set up a network with 6 nodes. let num_nodes = 6; // set epoch length to 30 mins so it never elapses unless we advance the clock From b4bed941c968c152ef3ec7fde8e9754646673f5f Mon Sep 17 00:00:00 2001 From: GTC6244 <95836911+GTC6244@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:01:18 -0500 Subject: [PATCH 4/5] closer --- rust/lit-node/lit-node-testnet/src/lib.rs | 8 ++++++++ rust/lit-node/lit-node-testnet/src/node_collection.rs | 2 +- .../lit-node-testnet/src/testnet/contracts_repo.rs | 2 +- rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs | 1 + 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/rust/lit-node/lit-node-testnet/src/lib.rs b/rust/lit-node/lit-node-testnet/src/lib.rs index 2c8b3c9..a75ebe8 100644 --- a/rust/lit-node/lit-node-testnet/src/lib.rs +++ b/rust/lit-node/lit-node-testnet/src/lib.rs @@ -37,6 +37,7 @@ pub struct TestSetupBuilder { wait_for_root_keys: bool, fund_wallet: bool, fund_ledger_for_wallet: bool, + custom_binary_path: Option, } impl Default for TestSetupBuilder { @@ -57,6 +58,7 @@ impl Default for TestSetupBuilder { wait_for_root_keys: true, fund_wallet: true, fund_ledger_for_wallet: true, + custom_binary_path: None, } } } @@ -140,6 +142,11 @@ impl TestSetupBuilder { self } + pub fn custom_binary_path(mut self, custom_binary_path: Option) -> Self { + self.custom_binary_path = custom_binary_path; + self + } + pub async fn build(self) -> (Testnet, ValidatorCollection, EndUser) { let node_keys_path = Path::new("./node_keys"); if node_keys_path.exists() { @@ -204,6 +211,7 @@ impl TestSetupBuilder { .wait_initial_epoch(self.wait_initial_epoch) .wait_for_root_keys(self.wait_for_root_keys) .node_binary_feature_flags(node_binary_feature_flags) + .custom_binary_path(self.custom_binary_path) .build(&testnet) .await .expect("Failed to build validator collection"); diff --git a/rust/lit-node/lit-node-testnet/src/node_collection.rs b/rust/lit-node/lit-node-testnet/src/node_collection.rs index 0671222..5a55e28 100644 --- a/rust/lit-node/lit-node-testnet/src/node_collection.rs +++ b/rust/lit-node/lit-node-testnet/src/node_collection.rs @@ -264,7 +264,7 @@ pub async fn get_node_versions(node_set: &Vec) -> Vec { // Parse response headers headers .iter() - .map(|header| header.get("X-Lit-Node-Version").unwrap().clone()) + .map(|header| header.get("x-lit-node-version").unwrap().clone()) .collect::>() } diff --git a/rust/lit-node/lit-node-testnet/src/testnet/contracts_repo.rs b/rust/lit-node/lit-node-testnet/src/testnet/contracts_repo.rs index bad2eea..2b12686 100644 --- a/rust/lit-node/lit-node-testnet/src/testnet/contracts_repo.rs +++ b/rust/lit-node/lit-node-testnet/src/testnet/contracts_repo.rs @@ -354,7 +354,7 @@ pub fn generate_wallet_and_add_as_alias() { "scripts/generate_wallet_and_add_as_alias.ts", ]; info!( - "Running full command in {}: npx {}", + "Running full generate_wallet_and_add_as_alias command in {}: npx {}", LITCONTRACTPATH, args.join(" ") ); diff --git a/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs b/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs index 5f9cab4..3ef165e 100644 --- a/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs +++ b/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs @@ -241,6 +241,7 @@ async fn test_version_upgrade_against_old_version(target_branch: &str) { let (testnet, mut validator_collection, end_user) = TestSetupBuilder::default() .num_staked_and_joined_validators(num_nodes) + .custom_binary_path(Some(old_build_path)) .force_deploy(true) .build() .await; From a30c9e99b6e4507fb41b627175afadf91c509320 Mon Sep 17 00:00:00 2001 From: GTC6244 <95836911+GTC6244@users.noreply.github.com> Date: Thu, 27 Nov 2025 00:03:04 -0500 Subject: [PATCH 5/5] Simple test that uses different stakers. --- rust/lit-node/lit-node-testnet/src/lib.rs | 14 +- .../lit-node-testnet/src/validator.rs | 9 + rust/lit-node/lit-node/src/git_info.rs | 2 +- .../tests/upgrades/version_upgrades.rs | 165 ++++++++++++++++-- 4 files changed, 168 insertions(+), 22 deletions(-) diff --git a/rust/lit-node/lit-node-testnet/src/lib.rs b/rust/lit-node/lit-node-testnet/src/lib.rs index a75ebe8..c6fc38d 100644 --- a/rust/lit-node/lit-node-testnet/src/lib.rs +++ b/rust/lit-node/lit-node-testnet/src/lib.rs @@ -38,6 +38,7 @@ pub struct TestSetupBuilder { fund_wallet: bool, fund_ledger_for_wallet: bool, custom_binary_path: Option, + start_staked_only_validators: bool, } impl Default for TestSetupBuilder { @@ -59,6 +60,7 @@ impl Default for TestSetupBuilder { fund_wallet: true, fund_ledger_for_wallet: true, custom_binary_path: None, + start_staked_only_validators: true, } } } @@ -82,6 +84,11 @@ impl TestSetupBuilder { self } + pub fn start_staked_only_validators(mut self, start_staked_only_validators: bool) -> Self { + self.start_staked_only_validators = start_staked_only_validators; + self + } + pub fn force_deploy(mut self, force_deploy: bool) -> Self { self.force_deploy = force_deploy; self @@ -197,8 +204,11 @@ impl TestSetupBuilder { } } - let num_staked_nodes = - self.num_staked_and_joined_validators + self.num_staked_only_validators; + let num_staked_nodes = if self.start_staked_only_validators { + self.num_staked_and_joined_validators + self.num_staked_only_validators + } else { + self.num_staked_and_joined_validators + }; let node_binary_feature_flags = if self.is_fault_test { "lit-actions,testing,proxy_chatter".to_string() diff --git a/rust/lit-node/lit-node-testnet/src/validator.rs b/rust/lit-node/lit-node-testnet/src/validator.rs index 0b8177d..7c12f2e 100644 --- a/rust/lit-node/lit-node-testnet/src/validator.rs +++ b/rust/lit-node/lit-node-testnet/src/validator.rs @@ -857,6 +857,15 @@ impl ValidatorCollection { }) .collect() } + pub async fn active_node_set(&self) -> Result> { + Ok(self.get_active_validators().await? + .iter() + .map(|v| NodeSet { + socket_address: v.public_address(), + value: 1, + }) + .collect::>()) + } } impl Drop for ValidatorCollection { diff --git a/rust/lit-node/lit-node/src/git_info.rs b/rust/lit-node/lit-node/src/git_info.rs index 9bcec2e..b547df5 100644 --- a/rust/lit-node/lit-node/src/git_info.rs +++ b/rust/lit-node/lit-node/src/git_info.rs @@ -1 +1 @@ -pub const GIT_COMMIT_HASH: &str = "3abee17a2f5384e7b08ba7ec446070e9124f61f4"; +pub const GIT_COMMIT_HASH: &str = "b4bed941c968c152ef3ec7fde8e9754646673f5f"; diff --git a/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs b/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs index 3ef165e..bbbca46 100644 --- a/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs +++ b/rust/lit-node/lit-node/tests/upgrades/version_upgrades.rs @@ -5,7 +5,7 @@ use lit_node_testnet::{ NodeAccount, Testnet, contracts_repo::{ self, WalletManifestItem, alias_node_configs_path, get_alias_manifest_template, - latest_wallet_manifest, save_alias_manifest, + latest_wallet_manifest, node_configs_path, save_alias_manifest, }, }, }; @@ -33,16 +33,18 @@ use tracing::info; async fn node_boot_invalid_version() { crate::common::setup_logging(); info!("TEST: node_boot_invalid_version"); - // Set up a network with 6 nodes. - let num_nodes = 6; // set epoch length to 30 mins so it never elapses unless we advance the clock - let epoch_length = 1800; - let (testnet, mut validator_collection, end_user) = TestSetupBuilder::default() - .num_staked_and_joined_validators(num_nodes) - .build() - .await; + let (testnet, mut validator_collection, end_user) = TestSetupBuilder::default().build().await; + let realm_id = U256::from(1); + let epoch_length = testnet + .actions() + .get_epoch_length(realm_id) + .await + .unwrap() + .as_u64() as usize; + let num_nodes = validator_collection.validator_count(); let actions = testnet.actions(); let network_checker = NetworkIntegrityChecker::new(&end_user, &actions).await; @@ -213,10 +215,7 @@ async fn active_validator_invalid_version() { async fn test_version_upgrade_against_old_version(target_branch: &str) { crate::common::setup_logging(); - info!( - "TEST: test_version_upgrade_against_old_version against {}", - target_branch - ); + info!("TEST: Upgrade against branch: {}", target_branch); // Get the commit hash that we want the build for. let old_build_commit_hash = @@ -235,25 +234,22 @@ async fn test_version_upgrade_against_old_version(target_branch: &str) { info!("TEST: test_version_upgrade_against_old_version"); // Set up a network with 6 nodes. - let num_nodes = 6; // set epoch length to 30 mins so it never elapses unless we advance the clock let epoch_length = 1800; let (testnet, mut validator_collection, end_user) = TestSetupBuilder::default() - .num_staked_and_joined_validators(num_nodes) .custom_binary_path(Some(old_build_path)) .force_deploy(true) .build() .await; + let num_nodes = validator_collection.validator_count(); let actions = testnet.actions(); let realm_id = U256::from(1); let starting_epoch = actions.get_current_epoch(realm_id).await; - let mut next_epoch = starting_epoch + 1; // Keep track of the node versions. - let complete_node_set = &validator_collection.complete_node_set(); let initial_node_versions = get_node_versions(&complete_node_set).await; info!("Initial node versions: {:?}", initial_node_versions); @@ -266,9 +262,6 @@ async fn test_version_upgrade_against_old_version(target_branch: &str) { let network_checker = NetworkIntegrityChecker::new(&end_user, &actions).await; network_checker.check(&validator_collection, &vec![]).await; - // network_checker - // .check_with_drained_presigns(&validator_collection) - // .await; // First, we shuffle the order of the original staker wallets that we will be gradually adding aliases for. let mut wallet_manifest_wallets = latest_wallet_manifest(false); @@ -392,6 +385,140 @@ async fn test_version_upgrade_against_old_version(target_branch: &str) { } } +/// This test assumes that you have the lit_node builds for the target branches. +/// During local development, there are two ways to get the builds: +/// 1. Run the `build_target_branches` script in the `scripts` directory. (x86 and arm64 builds) +/// 2. Run the `download_builds` script in the `scripts` directory. (x86 builds only) +/// The test will fail if the builds are not found. +#[test_case("origin/release-naga-prod-2025-11-25"; "Upgrade against the latest NAGA-Prod release branch")] +#[tokio::test] +async fn test_version_upgrade_against_old_version_with_new_stakers(target_branch: &str) { + crate::common::setup_logging(); + + info!("TEST: Upgrade against (new): {}", target_branch); + + // Get the commit hash that we want the build for. + let old_build_commit_hash = + utils::get_target_branch_commit_hash(target_branch).expect("Failed to get commit hash"); + + info!("Old build commit hash: {}", old_build_commit_hash); + // First check if we have the build. + let old_build_path = format!("./target/test-run/debug/lit_node_{}", old_build_commit_hash); + assert!( + fs::metadata(&old_build_path).is_ok(), + "Build does not exist at {}", + old_build_path + ); + + let num_nodes = 5; + // Set up a network of nodes running the old build. + + info!("TEST: test_version_upgrade_against_old_version"); + // Set up a network with 6 nodes. + // set epoch length to 30 mins so it never elapses unless we advance the clock + let epoch_length = 1800; + + let (testnet, mut validator_collection, end_user) = TestSetupBuilder::default() + .custom_binary_path(Some(old_build_path)) + // .force_deploy(true) // make sure all the initial chain data comes from the old build. + .num_staked_and_joined_validators(num_nodes) + .num_staked_only_validators(num_nodes) + .start_staked_only_validators(false) + .build() + .await; + + let num_nodes = validator_collection.validator_count(); + let actions = testnet.actions(); + let realm_id = U256::from(1); + let starting_epoch = actions.get_current_epoch(realm_id).await; + let mut next_epoch = starting_epoch + 1; + + let current_crate_version = get_crate_version(); + + // Keep track of the node versions. + let complete_node_set = &validator_collection.complete_node_set(); + let initial_node_versions = get_node_versions(&complete_node_set).await; + info!("Initial node versions: {:?}", initial_node_versions); + // Assert all node versions are the same. + assert!( + initial_node_versions + .iter() + .all(|v| v == &initial_node_versions[0]) + ); + + let network_checker = NetworkIntegrityChecker::new(&end_user, &actions).await; + network_checker.check(&validator_collection, &vec![]).await; + + // Keep dealing in new node versions and dealing out old node versions until the entire network is upgraded. + for upgrade_round in 0..num_nodes { + let validator = validator_collection.get_validator_by_idx(upgrade_round); + info!( + "Requesting to leave the network for staker {:?}", + validator.account().staker_address + ); + assert!(validator.request_to_leave(&actions).await.is_ok()); + + info!("Upgrading node {} to the new build", upgrade_round); + + let node_account = &testnet.node_accounts[upgrade_round + num_nodes]; + let validator_idx = upgrade_round + num_nodes; + let node_config_file_path = + format!("{}/lit_config{:?}.toml", node_configs_path(), validator_idx); + + assert!( + validator_collection + .add_one_custom( + false, + node_config_file_path, + node_account, + Some(lit_node_testnet::validator::BuildMode::UseNewOrCachedBuild), + 1 + ) + .await + .is_ok() + ); + + // Fast forward time to allow nodes to start a DKG to advance to the next epoch. + actions.increase_blockchain_timestamp(epoch_length).await; + + // After next epoch arrives, run interpolation and decryption tests. + actions.wait_for_epoch(realm_id, next_epoch).await; + next_epoch += U256::from(1); + + actions.sleep_millis(2000).await; // FIXME : let the nodes all acknowledge the epoch, then run the tests. This should be removed once signing across epochs works. + + // Run network checks. + network_checker.check(&validator_collection, &vec![]).await; + + // Assert node versions. + let active_node_set = validator_collection + .active_node_set() + .await + .expect("Failed to get active node set"); + let mut node_versions = get_node_versions(&active_node_set).await; + // Sort the node versions to make it easier to compare. + node_versions.sort(); + info!( + "node versions ({:?}) {:?} and initial node versions {:?}", + node_versions.len(), + node_versions, + initial_node_versions + ); + assert_eq!(node_versions.len(), num_nodes); + + let initial_crate_count = node_versions + .iter() + .filter(|v| *v == &initial_node_versions[0]) + .count(); + let current_crate_count = node_versions + .iter() + .filter(|v| *v == ¤t_crate_version) + .count(); + assert_eq!(current_crate_count, upgrade_round + 1); + assert_eq!(initial_crate_count, num_nodes - upgrade_round - 1); + } +} + fn get_latest_alias_node_account(idx: usize, testnet: &Testnet) -> NodeAccount { let latest_alias_wallet_manifest = latest_wallet_manifest(true); let provider = ENDPOINT_MANAGER