diff --git a/Cargo.lock b/Cargo.lock index e2dbd227..cbff0bc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,6 +85,21 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.8" @@ -751,6 +766,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "chrono" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.52.0", +] + [[package]] name = "cipher" version = "0.4.4" @@ -2045,6 +2074,29 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -4429,6 +4481,7 @@ dependencies = [ name = "polkadot-whois" version = "0.2.35" dependencies = [ + "chrono", "clap", "color-eyre", "env_logger", diff --git a/essentials/src/api/api_client.rs b/essentials/src/api/api_client.rs index 669bffa0..63413252 100644 --- a/essentials/src/api/api_client.rs +++ b/essentials/src/api/api_client.rs @@ -229,9 +229,17 @@ impl> ApiClient { self.storage().at_latest().await?.fetch(&addr).await } - pub async fn get_session_account_keys(&self, session_index: u32) -> Result>, subxt::Error> { + pub async fn get_session_account_keys( + &self, + session_index: u32, + hash: Option, + ) -> Result>, subxt::Error> { let addr = polkadot::storage().para_session_info().account_keys(session_index); - self.storage().at_latest().await?.fetch(&addr).await + if let Some(hash) = hash { + self.storage().at(hash).fetch(&addr).await + } else { + self.storage().at_latest().await?.fetch(&addr).await + } } pub async fn get_session_next_keys(&self, account: &AccountId32) -> Result, subxt::Error> { diff --git a/essentials/src/api/executor.rs b/essentials/src/api/executor.rs index d08f1ff3..a9d8a62d 100644 --- a/essentials/src/api/executor.rs +++ b/essentials/src/api/executor.rs @@ -60,7 +60,7 @@ pub enum Request { GetOccupiedCores(H256), GetBackingGroups(H256), GetSessionIndex(H256), - GetSessionAccountKeys(u32), + GetSessionAccountKeys(u32, Option), GetSessionNextKeys(AccountId32), GetSessionQueuedKeys(Option), GetInboundOutBoundHrmpChannels(H256, Vec), @@ -233,8 +233,8 @@ impl RequestExecutorBackend { BackingGroups(decode_validator_groups(&value)?) }, GetSessionIndex(hash) => SessionIndex(client.get_session_index(hash).await?.unwrap_or_default()), - GetSessionAccountKeys(session_index) => - SessionAccountKeys(client.get_session_account_keys(session_index).await?), + GetSessionAccountKeys(session_index, at) => + SessionAccountKeys(client.get_session_account_keys(session_index, at).await?), GetSessionNextKeys(ref account) => SessionNextKeys(client.get_session_next_keys(account).await?), GetSessionQueuedKeys(at) => SessionQueuedKeys(client.get_session_queued_keys(at).await?), GetSessionIndexNow => SessionIndex(client.get_session_index_now().await?.unwrap_or_default()), @@ -431,8 +431,9 @@ impl RequestExecutor { &mut self, url: &str, session_index: u32, + at: Option, ) -> color_eyre::Result>, RequestExecutorError> { - wrap_backend_call!(self, url, GetSessionAccountKeys, SessionAccountKeys, session_index) + wrap_backend_call!(self, url, GetSessionAccountKeys, SessionAccountKeys, session_index, at) } pub async fn get_session_next_keys( diff --git a/essentials/src/collector/mod.rs b/essentials/src/collector/mod.rs index 2b7ee4df..dacdbe5c 100644 --- a/essentials/src/collector/mod.rs +++ b/essentials/src/collector/mod.rs @@ -608,7 +608,7 @@ impl Collector { debug!("new session: {}, hash: {}", cur_session, cur_session_hash); let accounts_keys = self .executor - .get_session_account_keys(self.endpoint.as_str(), cur_session) + .get_session_account_keys(self.endpoint.as_str(), cur_session, None) .await? .ok_or_else(|| eyre!("Missing account keys for session {}", cur_session))?; self.storage_write_prefixed( diff --git a/whois/Cargo.toml b/whois/Cargo.toml index 3cd745c4..2c3a6e11 100644 --- a/whois/Cargo.toml +++ b/whois/Cargo.toml @@ -15,6 +15,7 @@ futures = { workspace = true } log = { workspace = true } polkadot-introspector-essentials = { workspace = true } polkadot-introspector-priority-channel = { workspace = true } +chrono = "0.4" thiserror = { workspace = true } tokio = { workspace = true } serde = {workspace = true } diff --git a/whois/src/main.rs b/whois/src/main.rs index 51a58387..4f2a8864 100644 --- a/whois/src/main.rs +++ b/whois/src/main.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with polkadot-introspector. If not, see . +use chrono::{DateTime, Utc}; use clap::{Args, Parser, Subcommand}; use futures::future; use itertools::Itertools; @@ -24,16 +25,21 @@ use polkadot_introspector_essentials::{ executor::{RequestExecutor, RequestExecutorError}, }, init, - types::{AccountId32, H256}, + metadata::{ + polkadot::session::events::new_session::SessionIndex, + polkadot_primitives::{AvailabilityBitfield, ValidatorIndex}, + }, + types::{AccountId32, SessionKeys, H256}, utils, }; use serde::{Deserialize, Serialize}; use serde_binary::binary_stream::Endian; use ss58_registry::Ss58AddressFormat; use std::{ - collections::{HashMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, fs::{self, File}, io::Write, + time::{Duration, UNIX_EPOCH}, }; use subp2p_explorer::util::{crypto::sr25519, p2p::get_peer_id}; use subp2p_explorer_cli::commands::authorities::PeerDetails; @@ -89,6 +95,35 @@ enum WhoisCommand { ByPeerId(PeerIdOptions), /// Display information about all validators. DumpAll, + /// Display performance statistics for bitfields for each validator. + BitFieldsPerformance(BitFieldsPerformance), +} +#[derive(Copy, Clone, Debug, Args)] +struct BitFieldsPerformance { + /// The block number to start from. + start_block: u32, + /// The step to go back in blocks. + num_to_skip: u32, + /// The number of blocks to check. + num_blocks: u32, + /// Print only validators that have at least this number of sessions with poor performance. + num_sessions_bellow_threshold: u32, + /// The threshold for poor performance. + poor_performance_threshold: f64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Default)] +struct BitfieldsStats { + // Number of blocks with poor performance. + num_poor_performance: usize, + // Number of blocks with the bitfield of the validator being present. + num_present: usize, +} + +impl BitfieldsStats { + fn percentage_poor(&self) -> f64 { + (self.num_poor_performance as f64 * 100.0) / self.num_present as f64 + } } #[derive(Clone, Debug, Args)] @@ -186,12 +221,14 @@ impl Whois { )); } - let para_session_account_keys = - match executor.get_session_account_keys(&self.opts.ws, self.opts.session_index).await { - Ok(Some(validators)) => validators, - Err(e) => return Err(WhoisError::SubxtError(e)), - _ => return Err(WhoisError::NoParaSessionAccountKeys), - }; + let para_session_account_keys = match executor + .get_session_account_keys(&self.opts.ws, self.opts.session_index, None) + .await + { + Ok(Some(validators)) => validators, + Err(e) => return Err(WhoisError::SubxtError(e)), + _ => return Err(WhoisError::NoParaSessionAccountKeys), + }; let session_queued_keys = match executor .get_session_queued_keys(&self.opts.ws, self.opts.queued_keys_at_block) @@ -272,6 +309,17 @@ impl Whois { .enumerate() .map(|(validator_index, account)| (account, Some(validator_index))) .collect_vec(), + WhoisCommand::BitFieldsPerformance(opts) => { + self.run_bitfields_performance_analysis( + opts, + &mut executor, + session_queued_keys, + network_cache_for_session, + ) + .await; + executor.close().await; + return Ok(vec![]); + }, }; let mut current_authority_discovery_keys = HashSet::new(); @@ -309,6 +357,215 @@ impl Whois { Ok(vec![]) } + + async fn run_bitfields_performance_analysis( + &self, + opts: BitFieldsPerformance, + executor: &mut RequestExecutor, + session_queued_keys: Vec<(AccountId32, SessionKeys)>, + network_cache_for_session: &PerSessionNetworkCache, + ) { + println!("Running bitfields performance analysis"); + let mut per_session_by_validatory_stats: BTreeMap> = + BTreeMap::new(); + let mut accounts_by_session = BTreeMap::new(); + let mut per_account_by_session_stats: BTreeMap> = + BTreeMap::new(); + let mut session_timestamps = BTreeMap::new(); + let mut start = opts.start_block; + let mut num_blocks_poor_perf_more_than_a_third = BTreeMap::new(); + + for _ in 0..opts.num_blocks { + let Ok(Some(block_hash)) = executor.get_block_hash(&self.opts.ws, Some(start)).await else { + break; + }; + + let Ok(session_index_now) = executor.get_session_index(&self.opts.ws, block_hash).await else { + break; + }; + + if !accounts_by_session.contains_key(&session_index_now) { + let Ok(timestamp) = executor.get_block_timestamp(&self.opts.ws, block_hash).await else { + break; + }; + + let timestamp_date = UNIX_EPOCH + Duration::from_millis(timestamp); + let timestamp_date: DateTime = DateTime::from(timestamp_date); + let timestamp_str = timestamp_date.format("%Y-%m-%d %H:%M").to_string(); + session_timestamps.insert(session_index_now, timestamp_str.clone()); + println!("session: {} timestamp: {}", session_index_now, timestamp_str); + let para_session_account_keys = match executor + .get_session_account_keys(&self.opts.ws, session_index_now, Some(block_hash)) + .await + { + Ok(Some(validators)) => validators, + Err(_e) => break, + _ => break, + }; + + let para_session_account_keys = para_session_account_keys + .into_iter() + .enumerate() + .map(|(validator_index, account)| (account, validator_index)) + .collect_vec(); + accounts_by_session.insert(session_index_now, para_session_account_keys); + } + + let Ok(para_inherent) = executor.extract_parainherent_data(&self.opts.ws, Some(block_hash)).await else { + break; + }; + + let bitfields = para_inherent + .bitfields + .into_iter() + .map(|b| (b.payload, b.validator_index)) + .collect::>(); + + let mut num_poor_performance_per_block = 0; + + let Some(max) = bitfields + .iter() + .map(|(bitfield, _)| bitfield.0.as_bits().iter().filter(|bit| *bit).count()) + .max() + else { + break; + }; + + let Some(session_accounts) = accounts_by_session.get(&session_index_now) else { + break; + }; + let session_stats = per_session_by_validatory_stats.entry(session_index_now).or_default(); + + for bitfield in bitfields { + let num_bits_set = bitfield.0 .0.as_bits().iter().filter(|bit| *bit).count(); + + let Some(validator) = session_accounts + .iter() + .find(|(_, validator)| *validator as u32 == bitfield.1 .0) + .map(|val| val.0.clone()) + else { + break; + }; + + let per_account_stats = per_account_by_session_stats + .entry(validator) + .or_default() + .entry(session_index_now) + .or_default(); + + let per_session_stats = session_stats.entry(bitfield.1 .0).or_insert(Default::default()); + + // The validator has poor performance if it has less than 2 bits set. + if num_bits_set < std::cmp::min(max, 2) { + num_poor_performance_per_block += 1; + (*per_session_stats).num_poor_performance += 1; + per_account_stats.num_poor_performance += 1; + } + (*per_session_stats).num_present += 1; + per_account_stats.num_present += 1; + } + + if num_poor_performance_per_block > session_accounts.len() / 3 { + *(num_blocks_poor_perf_more_than_a_third.entry(session_index_now).or_insert(0)) += 1; + } + start -= opts.num_to_skip; + } + + print_per_session_performance(opts, per_session_by_validatory_stats, &session_timestamps); + + print_per_account_performance( + opts, + session_queued_keys, + network_cache_for_session, + per_account_by_session_stats, + &session_timestamps, + ); + + for (session, more_than_a_third) in num_blocks_poor_perf_more_than_a_third.iter() { + println!( + "block_start: {}, session: {}: {}, Count blocks with unincluded {:}", + opts.start_block, + session, + session_timestamps.get(session).cloned().unwrap_or_default(), + more_than_a_third, + ); + } + } +} + +fn print_per_account_performance( + opts: BitFieldsPerformance, + session_queued_keys: Vec<(AccountId32, SessionKeys)>, + network_cache_for_session: &PerSessionNetworkCache, + per_account_by_session_stats: BTreeMap>, + session_timestamps: &BTreeMap, +) { + for (account, account_stats) in per_account_by_session_stats.iter() { + let count_past_30 = account_stats + .iter() + .map(|(_, stats)| (stats.num_poor_performance as f64 * 100.0) / stats.num_present as f64) + .filter(|x| *x > opts.poor_performance_threshold) + .count(); + + if count_past_30 < opts.num_sessions_bellow_threshold as usize { + continue; + } + + let session_keys_for_validator = &session_queued_keys.iter().find(|(this_account, _)| this_account == account); + + let name = if let Some((_, session_keys_for_validator)) = session_keys_for_validator { + let authority_discovery_key = session_keys_for_validator.authority_discovery.0; + let (_, info, _) = network_cache_for_session.get_details(authority_discovery_key); + info + } else { + "unknown".to_string() + }; + + println!("block_start: {}, account: {} name: {}", opts.start_block, account, name); + for (session, stats) in account_stats.iter() { + let percentage = stats.percentage_poor(); + + println!( + " at {} session: {}, has {:?} with zero bits, {:.2}% percent", + session_timestamps.get(session).cloned().unwrap_or_default(), + session, + stats.num_poor_performance, + percentage + ); + } + } +} + +fn print_per_session_performance( + opts: BitFieldsPerformance, + per_session_by_validatory_stats: BTreeMap>, + session_timestamps: &BTreeMap, +) { + let mut count_per_session_poor_performance = BTreeMap::new(); + for (session, stats_by_validator) in per_session_by_validatory_stats.iter() { + for (validator, stats) in stats_by_validator.iter() { + let percentage = stats.percentage_poor(); + if percentage > opts.poor_performance_threshold { + *(count_per_session_poor_performance.entry(session).or_insert(0)) += 1; + } + + println!( + "block_start: {}, session: {}: validator: {:?} has {:?} zero bits {:.3}%", + opts.start_block, session, validator, stats.num_poor_performance, percentage + ); + } + } + + for (session, count) in count_per_session_poor_performance.iter() { + println!( + "block_start: {}, session: {}: {} Number of validators with 10% missing {:} count_all_with_0 {:}", + opts.start_block, + session, + session_timestamps.get(session).cloned().unwrap_or_default(), + count, + per_session_by_validatory_stats.len() + ); + } } const DEFAULT_CACHE_DIR: &str = "whois_p2pcache";