From 7b01f5d65be4087c76110a345c6d28a8548c8389 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 15 Jul 2024 16:49:51 +0800 Subject: [PATCH 001/112] Implement JumpDest fetching from RPC. Supports CREATE1/CREATE2 Supports transaction deployments Includes test scripts for randomised testing --- .gitignore | 1 + Cargo.lock | 4 + Cargo.toml | 4 +- evm_arithmetization/Cargo.toml | 1 + .../benches/fibonacci_25m_gas.rs | 1 + .../src/cpu/kernel/interpreter.rs | 60 ++- .../src/cpu/kernel/tests/add11.rs | 2 + .../kernel/tests/core/jumpdest_analysis.rs | 24 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 1 + .../src/cpu/kernel/tests/mod.rs | 2 +- .../src/generation/jumpdest.rs | 203 +++++++++ evm_arithmetization/src/generation/mod.rs | 11 + .../src/generation/prover_input.rs | 48 +- evm_arithmetization/src/generation/state.rs | 3 +- evm_arithmetization/src/lib.rs | 3 + evm_arithmetization/tests/add11_yml.rs | 1 + evm_arithmetization/tests/erc20.rs | 1 + evm_arithmetization/tests/erc721.rs | 1 + evm_arithmetization/tests/global_exit_root.rs | 1 + evm_arithmetization/tests/log_opcode.rs | 1 + evm_arithmetization/tests/selfdestruct.rs | 1 + evm_arithmetization/tests/simple_transfer.rs | 1 + evm_arithmetization/tests/withdrawals.rs | 1 + test_jerigon.sh | 88 ++++ test_native.sh | 83 ++++ trace_decoder/src/core.rs | 12 + trace_decoder/src/interface.rs | 20 +- trace_decoder/src/jumpdest.rs | 1 + trace_decoder/src/lib.rs | 1 + zero_bin/prover/src/cli.rs | 2 +- zero_bin/rpc/Cargo.toml | 3 + zero_bin/rpc/src/jerigon.rs | 134 +++++- zero_bin/rpc/src/jumpdest.rs | 417 ++++++++++++++++++ zero_bin/rpc/src/lib.rs | 1 + zero_bin/rpc/src/main.rs | 3 + zero_bin/rpc/src/native/mod.rs | 6 +- zero_bin/rpc/src/native/txn.rs | 72 ++- zero_bin/tools/prove_stdio.sh | 22 +- 38 files changed, 1170 insertions(+), 71 deletions(-) create mode 100644 evm_arithmetization/src/generation/jumpdest.rs create mode 100755 test_jerigon.sh create mode 100755 test_native.sh create mode 100644 trace_decoder/src/jumpdest.rs create mode 100644 zero_bin/rpc/src/jumpdest.rs diff --git a/.gitignore b/.gitignore index f035f7002..dd5a27331 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ .vscode /**/*.ignoreme **/output.log +witnesses diff --git a/Cargo.lock b/Cargo.lock index 1791c16ed..0b4835ef6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2066,6 +2066,7 @@ dependencies = [ "plonky2", "plonky2_maybe_rayon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "plonky2_util", + "primitive-types 0.12.2", "rand", "rand_chacha", "ripemd", @@ -4169,6 +4170,8 @@ version = "0.1.0" dependencies = [ "alloy", "alloy-compat", + "alloy-primitives", + "alloy-serde", "anyhow", "cargo_metadata", "clap", @@ -4177,6 +4180,7 @@ dependencies = [ "futures", "hex", "itertools 0.13.0", + "keccak-hash 0.10.0", "mpt_trie", "primitive-types 0.12.2", "proof_gen", diff --git a/Cargo.toml b/Cargo.toml index 9628a345c..84a438806 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,8 @@ alloy = { version = '0.3.0', default-features = false, features = [ "transport-http", "rpc-types-debug", ] } +alloy-primitives = "0.8.0" +alloy-serde = "0.3.0" anyhow = "1.0.86" async-stream = "0.3.5" axum = "0.7.5" @@ -57,7 +59,6 @@ criterion = "0.5.1" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" -enumn = "0.1.13" env_logger = "0.11.3" eth_trie = "0.4.0" ethereum-types = "0.14.1" @@ -94,7 +95,6 @@ serde = "1.0.203" serde-big-array = "0.5.1" serde_json = "1.0.118" serde_path_to_error = "0.1.16" -serde_with = "3.8.1" sha2 = "0.10.8" static_assertions = "1.1.0" thiserror = "1.0.61" diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index ffdb8d2f5..e8e832c2d 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -15,6 +15,7 @@ homepage.workspace = true keywords.workspace = true [dependencies] +__compat_primitive_types = { workspace = true } anyhow = { workspace = true } bytes = { workspace = true } env_logger = { workspace = true } diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index f6d5fc39f..57b902c63 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -193,6 +193,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 10af8d495..d0cb867fa 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -5,10 +5,12 @@ //! the future execution and generate nondeterministically the corresponding //! jumpdest table, before the actual CPU carries on with contract execution. +use core::option::Option::None; use std::collections::{BTreeMap, BTreeSet, HashMap}; use anyhow::anyhow; use ethereum_types::{BigEndianHash, U256}; +use keccak_hash::H256; use log::Level; use mpt_trie::partial_trie::PartialTrie; use plonky2::hash::hash_types::RichField; @@ -19,7 +21,9 @@ use crate::cpu::columns::CpuColumnsView; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; use crate::generation::debug_inputs; +use crate::generation::jumpdest::{ContextJumpDests, JumpDestTableProcessed, JumpDestTableWitness}; use crate::generation::mpt::{load_linked_lists_and_txn_and_receipt_mpts, TrieRootPtrs}; +use crate::generation::prover_input::get_proofs_and_jumpdests; use crate::generation::rlp::all_rlp_prover_inputs_reversed; use crate::generation::state::{ all_ger_prover_inputs, all_withdrawals_prover_inputs_reversed, GenerationState, @@ -56,6 +60,7 @@ pub(crate) struct Interpreter { /// Counts the number of appearances of each opcode. For debugging purposes. #[allow(unused)] pub(crate) opcode_count: [usize; 0x100], + /// A table of contexts and their reached JUMPDESTs. jumpdest_table: HashMap>, /// `true` if the we are currently carrying out a jumpdest analysis. pub(crate) is_jumpdest_analysis: bool, @@ -71,9 +76,9 @@ pub(crate) struct Interpreter { pub(crate) fn simulate_cpu_and_get_user_jumps( final_label: &str, state: &GenerationState, -) -> Option>> { +) -> Option<(JumpDestTableProcessed, JumpDestTableWitness)> { match state.jumpdest_table { - Some(_) => None, + Some(_) => Default::default(), None => { let halt_pc = KERNEL.global_labels[final_label]; let initial_context = state.registers.context; @@ -92,16 +97,15 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let clock = interpreter.get_clock(); - interpreter + let jdtw = interpreter .generation_state - .set_jumpdest_analysis_inputs(interpreter.jumpdest_table); + .set_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); log::debug!( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - - interpreter.generation_state.jumpdest_table + (interpreter.generation_state.jumpdest_table).map(|x| (x, jdtw)) } } } @@ -114,9 +118,9 @@ pub(crate) struct ExtraSegmentData { pub(crate) withdrawal_prover_inputs: Vec, pub(crate) ger_prover_inputs: Vec, pub(crate) trie_root_ptrs: TrieRootPtrs, - pub(crate) jumpdest_table: Option>>, pub(crate) accounts: BTreeMap, pub(crate) storage: BTreeMap<(U256, U256), usize>, + pub(crate) jumpdest_table: Option, pub(crate) next_txn_index: usize, } @@ -150,6 +154,48 @@ pub(crate) fn set_registers_and_run( interpreter.run() } +/// Computes the JUMPDEST proofs for each context. +/// +/// # Arguments +/// +/// - `jumpdest_table_rpc`: The raw table received from RPC. +/// - `code_db`: The corresponding database of contract code used in the trace. +pub(crate) fn set_jumpdest_analysis_inputs_rpc( + jumpdest_table_rpc: &JumpDestTableWitness, + code_map: &HashMap>, +) -> JumpDestTableProcessed { + let ctx_proofs = (*jumpdest_table_rpc) + .iter() + .flat_map(|(code_addr, ctx_jumpdests)| { + prove_context_jumpdests(&code_map[code_addr], ctx_jumpdests) + }) + .collect(); + JumpDestTableProcessed::new(ctx_proofs) +} + +/// Orchestrates the proving of all contexts in a specific bytecode. +/// +/// # Arguments +/// +/// - `ctx_jumpdests`: Map from `ctx` to its list of offsets to reached +/// `JUMPDEST`s. +/// - `code`: The bytecode for the contexts. This is the same for all contexts. +fn prove_context_jumpdests( + code: &[u8], + ctx_jumpdests: &ContextJumpDests, +) -> HashMap> { + ctx_jumpdests + .0 + .iter() + .map(|(&ctx, jumpdests)| { + let proofs = jumpdests.last().map_or(Vec::default(), |&largest_address| { + get_proofs_and_jumpdests(code, largest_address, jumpdests.clone()) + }); + (ctx, proofs) + }) + .collect() +} + impl Interpreter { /// Returns an instance of `Interpreter` given `GenerationInputs`, and /// assuming we are initializing with the `KERNEL` code. diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index fe36b40e3..1cc5588e1 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -194,6 +194,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let initial_stack = vec![]; @@ -371,6 +372,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index b0ef17033..1d93e0e31 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -7,6 +7,7 @@ use plonky2::field::goldilocks_field::GoldilocksField as F; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::interpreter::Interpreter; use crate::cpu::kernel::opcodes::{get_opcode, get_push_opcode}; +use crate::generation::jumpdest::JumpDestTableProcessed; use crate::witness::operation::CONTEXT_SCALING_FACTOR; #[test] @@ -67,7 +68,10 @@ fn test_jumpdest_analysis() -> Result<()> { interpreter.generation_state.jumpdest_table, // Context 3 has jumpdest 1, 5, 7. All have proof 0 and hence // the list [proof_0, jumpdest_0, ... ] is [0, 1, 0, 5, 0, 7, 8, 40] - Some(HashMap::from([(3, vec![0, 1, 0, 5, 0, 7, 8, 40])])) + Some(JumpDestTableProcessed::new(HashMap::from([( + 3, + vec![0, 1, 0, 5, 0, 7, 8, 40] + )]))) ); // Run jumpdest analysis with context = 3 @@ -84,14 +88,14 @@ fn test_jumpdest_analysis() -> Result<()> { // We need to manually pop the jumpdest_table and push its value on the top of // the stack - interpreter + (*interpreter .generation_state .jumpdest_table .as_mut() - .unwrap() - .get_mut(&CONTEXT) - .unwrap() - .pop(); + .unwrap()) + .get_mut(&CONTEXT) + .unwrap() + .pop(); interpreter .push(41.into()) .expect("The stack should not overflow"); @@ -136,7 +140,9 @@ fn test_packed_verification() -> Result<()> { let mut interpreter: Interpreter = Interpreter::new(write_table_if_jumpdest, initial_stack.clone(), None); interpreter.set_code(CONTEXT, code.clone()); - interpreter.generation_state.jumpdest_table = Some(HashMap::from([(3, vec![1, 33])])); + interpreter.generation_state.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from( + [(3, vec![1, 33])], + ))); interpreter.run()?; @@ -149,7 +155,9 @@ fn test_packed_verification() -> Result<()> { let mut interpreter: Interpreter = Interpreter::new(write_table_if_jumpdest, initial_stack.clone(), None); interpreter.set_code(CONTEXT, code.clone()); - interpreter.generation_state.jumpdest_table = Some(HashMap::from([(3, vec![1, 33])])); + interpreter.generation_state.jumpdest_table = Some(JumpDestTableProcessed::new( + HashMap::from([(3, vec![1, 33])]), + )); assert!(interpreter.run().is_err()); diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index be88021de..08fe57b1a 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,6 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, + jumpdest_tables: vec![], }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/mod.rs b/evm_arithmetization/src/cpu/kernel/tests/mod.rs index 39810148b..08b17d495 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/mod.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/mod.rs @@ -249,7 +249,7 @@ impl Interpreter { } pub(crate) fn set_jumpdest_analysis_inputs(&mut self, jumps: HashMap>) { - self.generation_state.set_jumpdest_analysis_inputs(jumps); + let _ = self.generation_state.set_jumpdest_analysis_inputs(jumps); } pub(crate) fn extract_kernel_memory(self, segment: Segment, range: Range) -> Vec { diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs new file mode 100644 index 000000000..fc2a8e1a6 --- /dev/null +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -0,0 +1,203 @@ +use std::cmp::max; +use std::{ + collections::{BTreeSet, HashMap}, + fmt::Display, + ops::{Deref, DerefMut}, +}; + +use itertools::{sorted, Itertools}; +use keccak_hash::H256; +use serde::{Deserialize, Serialize}; + +/// Each `CodeAddress` can be called one or more times, each time creating a new +/// `Context`. Each `Context` will contain one or more offsets of `JUMPDEST`. +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct ContextJumpDests(pub HashMap>); + +/// The result after proving a `JumpDestTableWitness`. +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub(crate) struct JumpDestTableProcessed(HashMap>); + +/// Map `CodeAddress -> (Context -> [JumpDests])` +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +pub struct JumpDestTableWitness(HashMap); + +impl ContextJumpDests { + pub fn insert(&mut self, ctx: usize, offset: usize) { + self.entry(ctx).or_default().insert(offset); + } + + pub fn get(&self, ctx: usize) -> Option<&BTreeSet> { + self.0.get(&ctx) + } +} + +impl JumpDestTableProcessed { + pub fn new(ctx_map: HashMap>) -> Self { + Self(ctx_map) + } +} + +impl JumpDestTableWitness { + pub fn new(ctx_map: HashMap) -> Self { + Self(ctx_map) + } + + pub fn get(&self, code_hash: &H256) -> Option<&ContextJumpDests> { + self.0.get(code_hash) + } + + /// Insert `offset` into `ctx` under the corresponding `code_hash`. + /// Creates the required `ctx` keys and `code_hash`. Idempotent. + pub fn insert(&mut self, code_hash: H256, ctx: usize, offset: usize) { + (*self).entry(code_hash).or_default().insert(ctx, offset); + + // TODO(einar) remove before publishing PR. + assert!(self.0.contains_key(&code_hash)); + assert!(self.0[&code_hash].0.contains_key(&ctx)); + assert!(self.0[&code_hash].0[&ctx].contains(&offset)); + } + + pub fn extend(mut self, other: &Self, prev_max_ctx: usize) -> (Self, usize) { + let mut curr_max_ctx = prev_max_ctx; + + // TODO: Opportunity for optimization: Simulate to generate only missing + // JUMPDEST tables. + for (code_hash, ctx_tbl) in (*other).iter() { + for (ctx, jumpdests) in ctx_tbl.0.iter() { + let batch_ctx = prev_max_ctx + ctx; + curr_max_ctx = max(curr_max_ctx, batch_ctx); + + for offset in jumpdests { + self.insert(*code_hash, batch_ctx, *offset); + + assert!(self.0.contains_key(code_hash)); + assert!(self.0[code_hash].0.contains_key(&batch_ctx)); + assert!(self.0[code_hash].0[&batch_ctx].contains(offset)); + } + // dbg!(&self); + } + } + + (self, curr_max_ctx) + } + + pub fn merge<'a>(jdts: impl IntoIterator) -> (Self, usize) { + jdts.into_iter() + .fold((Default::default(), 0), |(acc, cnt), t| acc.extend(t, cnt)) + } +} + +impl Display for JumpDestTableWitness { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "=== JumpDestTableWitness ===")?; + + for (code, ctxtbls) in &self.0 { + write!(f, "codehash: {:#x}\n{}", code, ctxtbls)?; + } + Ok(()) + } +} + +impl Display for ContextJumpDests { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let v: Vec<_> = self.0.iter().sorted().collect(); + for (ctx, offsets) in v.into_iter() { + write!(f, " ctx: {:>4}: [", ctx)?; + for offset in offsets { + write!(f, "{:#}, ", offset)?; + } + writeln!(f, "]")?; + } + Ok(()) + } +} + +impl Display for JumpDestTableProcessed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "=== JumpDestTableProcessed ===")?; + + let v = sorted(self.0.clone()); + for (ctx, code) in v { + writeln!(f, "ctx: {:?} {:?}", ctx, code)?; + } + Ok(()) + } +} + +impl Deref for ContextJumpDests { + type Target = HashMap>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for ContextJumpDests { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Deref for JumpDestTableProcessed { + type Target = HashMap>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for JumpDestTableProcessed { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Deref for JumpDestTableWitness { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for JumpDestTableWitness { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +#[cfg(test)] +mod test { + use keccak_hash::H256; + + use super::JumpDestTableWitness; + + #[test] + fn test_extend() { + let code_hash = H256::default(); + + let mut table1 = JumpDestTableWitness::default(); + table1.insert(code_hash, 1, 1); + table1.insert(code_hash, 2, 2); + table1.insert(code_hash, 42, 3); + table1.insert(code_hash, 43, 4); + let table2 = table1.clone(); + + let jdts = [&table1, &table2]; + let (actual, max_ctx) = JumpDestTableWitness::merge(jdts); + + let mut expected = JumpDestTableWitness::default(); + expected.insert(code_hash, 1, 1); + expected.insert(code_hash, 2, 2); + expected.insert(code_hash, 42, 3); + expected.insert(code_hash, 43, 4); + expected.insert(code_hash, 44, 1); + expected.insert(code_hash, 45, 2); + expected.insert(code_hash, 85, 3); + expected.insert(code_hash, 86, 4); + + assert_eq!(86, max_ctx); + assert_eq!(expected, actual) + } +} diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 9c7625d2b..87d9ed1f7 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use anyhow::anyhow; use ethereum_types::{Address, BigEndianHash, H256, U256}; +use jumpdest::JumpDestTableWitness; use keccak_hash::keccak; use log::log_enabled; use mpt_trie::partial_trie::{HashedPartialTrie, PartialTrie}; @@ -32,6 +33,7 @@ use crate::util::{h2u, u256_to_usize}; use crate::witness::memory::{MemoryAddress, MemoryChannel, MemoryState}; use crate::witness::state::RegistersState; +pub mod jumpdest; pub(crate) mod linked_list; pub mod mpt; pub(crate) mod prover_input; @@ -106,6 +108,10 @@ pub struct GenerationInputs { /// /// This is specific to `cdk-erigon`. pub ger_data: Option<(H256, H256)>, + + /// A table listing each JUMPDESTs reached in each call context under + /// associated code hash. + pub jumpdest_tables: Vec>, } /// A lighter version of [`GenerationInputs`], which have been trimmed @@ -156,6 +162,10 @@ pub struct TrimmedGenerationInputs { /// The hash of the current block, and a list of the 256 previous block /// hashes. pub block_hashes: BlockHashes, + + /// A list of tables listing each JUMPDESTs reached in each call context + /// under associated code hash. + pub jumpdest_tables: Vec>, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] @@ -230,6 +240,7 @@ impl GenerationInputs { burn_addr: self.burn_addr, block_metadata: self.block_metadata.clone(), block_hashes: self.block_hashes.clone(), + jumpdest_tables: self.jumpdest_tables.clone(), } } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 16c1e5310..ae49366b3 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -6,10 +6,12 @@ use std::str::FromStr; use anyhow::{bail, Error, Result}; use ethereum_types::{BigEndianHash, H256, U256, U512}; use itertools::Itertools; +use keccak_hash::keccak; use num_bigint::BigUint; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; +use super::jumpdest::{JumpDestTableProcessed, JumpDestTableWitness}; use super::linked_list::{ LinkedList, ACCOUNTS_LINKED_LIST_NODE_SIZE, STORAGE_LINKED_LIST_NODE_SIZE, }; @@ -20,7 +22,9 @@ use crate::cpu::kernel::constants::cancun_constants::{ POINT_EVALUATION_PRECOMPILE_RETURN_VALUE, }; use crate::cpu::kernel::constants::context_metadata::ContextMetadata; -use crate::cpu::kernel::interpreter::simulate_cpu_and_get_user_jumps; +use crate::cpu::kernel::interpreter::{ + set_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps, +}; use crate::curve_pairings::{bls381, CurveAff, CyclicGroup}; use crate::extension_tower::{FieldExt, Fp12, Fp2, BLS381, BLS_BASE, BLS_SCALAR, BN254, BN_BASE}; use crate::generation::prover_input::EvmField::{ @@ -38,6 +42,10 @@ use crate::witness::memory::MemoryAddress; use crate::witness::operation::CONTEXT_SCALING_FACTOR; use crate::witness::util::{current_context_peek, stack_peek}; +/// A set of contract code as a byte arrays. From this a mapping: hash -> +/// contract can be built. +pub type CodeDb = BTreeSet>; + /// Prover input function represented as a scoped function name. /// Example: `PROVER_INPUT(ff::bn254_base::inverse)` is represented as /// `ProverInputFn([ff, bn254_base, inverse])`. @@ -364,12 +372,12 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) + if let Some(ctx_jumpdest_table) = (*jumpdest_table).get_mut(&context) && let Some(next_jumpdest_address) = ctx_jumpdest_table.pop() { Ok((next_jumpdest_address + 1).into()) } else { - jumpdest_table.remove(&context); + (*jumpdest_table).remove(&context); Ok(U256::zero()) } } @@ -383,7 +391,7 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) + if let Some(ctx_jumpdest_table) = (*jumpdest_table).get_mut(&context) && let Some(next_jumpdest_proof) = ctx_jumpdest_table.pop() { Ok(next_jumpdest_proof.into()) @@ -757,7 +765,22 @@ impl GenerationState { fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - self.jumpdest_table = simulate_cpu_and_get_user_jumps("terminate_common", self); + + dbg!(&self.inputs.jumpdest_tables); + eprintln!("Generating JUMPDEST tables"); + // w for witness + let txn_idx = self.next_txn_index - 1; + let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref(); + let rpc = rpcw.map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + + if let Some((_sim, simw)) = simulate_cpu_and_get_user_jumps("terminate_common", self) { + if rpcw.is_some() && rpcw.unwrap() != &simw.clone() { + println!("SIMW {}", simw.clone()); + println!("RPCW {}", rpcw.unwrap()); + assert_eq!(simw.clone(), *rpcw.unwrap()); + } + } + self.jumpdest_table = rpc; Ok(()) } @@ -768,18 +791,23 @@ impl GenerationState { pub(crate) fn set_jumpdest_analysis_inputs( &mut self, jumpdest_table: HashMap>, - ) { - self.jumpdest_table = Some(HashMap::from_iter(jumpdest_table.into_iter().map( - |(ctx, jumpdest_table)| { + ) -> JumpDestTableWitness { + let mut jdtw = JumpDestTableWitness::default(); + self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( + jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); + for offset in jumpdest_table.clone() { + jdtw.insert(keccak(code.clone()), ctx, offset); + } if let Some(&largest_address) = jumpdest_table.last() { let proofs = get_proofs_and_jumpdests(&code, largest_address, jumpdest_table); (ctx, proofs) } else { (ctx, vec![]) } - }, + }), ))); + jdtw } pub(crate) fn get_current_code(&self) -> Result, ProgramError> { @@ -850,7 +878,7 @@ impl GenerationState { /// for which none of the previous 32 bytes in the code (including opcodes /// and pushed bytes) is a PUSHXX and the address is in its range. It returns /// a vector of even size containing proofs followed by their addresses. -fn get_proofs_and_jumpdests( +pub(crate) fn get_proofs_and_jumpdests( code: &[u8], largest_address: usize, jumpdest_table: std::collections::BTreeSet, diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index b094c15f9..39744ddf2 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -8,6 +8,7 @@ use keccak_hash::keccak; use log::Level; use plonky2::hash::hash_types::RichField; +use super::jumpdest::JumpDestTableProcessed; use super::linked_list::{AccountsLinkedList, StorageLinkedList}; use super::mpt::TrieRootPtrs; use super::segments::GenerationSegmentData; @@ -374,7 +375,7 @@ pub struct GenerationState { /// "proof" for a jump destination is either 0 or an address i > 32 in /// the code (not necessarily pointing to an opcode) such that for every /// j in [i, i+32] it holds that code[j] < 0x7f - j + i. - pub(crate) jumpdest_table: Option>>, + pub(crate) jumpdest_table: Option, /// Each entry contains the pair (key, ptr) where key is the (hashed) key /// of an account in the accounts linked list, and ptr is the respective diff --git a/evm_arithmetization/src/lib.rs b/evm_arithmetization/src/lib.rs index ca2cf2bd7..f813b18cf 100644 --- a/evm_arithmetization/src/lib.rs +++ b/evm_arithmetization/src/lib.rs @@ -210,6 +210,9 @@ pub mod verifier; pub mod generation; pub mod witness; +pub use generation::jumpdest; +pub use generation::prover_input::CodeDb; + // Utility modules pub mod curve_pairings; pub mod extension_tower; diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 5b8290229..51ac662df 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,6 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index 2c1a56829..f13f541b4 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -196,6 +196,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index 170a19df4..6386dbe69 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -200,6 +200,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/global_exit_root.rs b/evm_arithmetization/tests/global_exit_root.rs index a41d79e6f..8c18b8e47 100644 --- a/evm_arithmetization/tests/global_exit_root.rs +++ b/evm_arithmetization/tests/global_exit_root.rs @@ -114,6 +114,7 @@ fn test_global_exit_root() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + batch_jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index 44453ae22..d9fe6f234 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -267,6 +267,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 7d8ecba5b..27d68af7b 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -171,6 +171,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index 7dbf91399..4109ae09c 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -163,6 +163,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index fcdd73f49..3dfe13e43 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -106,6 +106,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, + jumpdest_tables: vec![], }; let max_cpu_len_log = 20; diff --git a/test_jerigon.sh b/test_jerigon.sh new file mode 100755 index 000000000..49f88ed12 --- /dev/null +++ b/test_jerigon.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +#export CARGO_LOG=cargo::core::compiler::fingerprint=debug +export RPC= +if [ -z $RPC ]; then + # You must set an RPC endpoint + exit 1 +fi +mkdir -p witnesses + +export RAYON_NUM_THREADS=4 +export TOKIO_WORKER_THREADS=4 +export RUST_BACKTRACE=full +export RUST_LOG=info +export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' +export RUST_MIN_STACK=33554432 + +GITHASH=`git rev-parse --short HEAD` +echo "Testing against jergion, current revision: $GITHASH." + +TESTNETBLOCKS=" +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +" + + +KNOWNFAILED=" +28 +444 +" + + + +# 470..663 from Robin +for i in {470..663} +do + ROBIN+=" $i" +done + +# Pick random blocks +for i in {1..10} +do + RANDOMBLOCKS+=" $((1 + $RANDOM % 688))" +done + +# TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +TESTNETBLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" + +TESTNETBLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` +SHUF=`shuf -e $TESTNETBLOCKS` +echo $SHUF + + + +for BLOCK in $TESTNETBLOCKS; do + GITHASH=`git rev-parse --short HEAD` + WITNESS="$witnesses/$BLOCK.jerigon.$GITHASH.witness.json" + echo "Fetching block $BLOCK" + cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + echo "Checking block $BLOCK" + zero_bin/tools/prove_stdio.sh $WITNESS test_only + EXITCODE=$? + if [ -n $EXITCODE ] + then + RESULT="success" + else + RESULT="failure" + fi + printf "%10i %s %s\n" $BLOCK $GITHASH $RESULT | tee -a result.txt +done + +exit 0 diff --git a/test_native.sh b/test_native.sh new file mode 100755 index 000000000..913f27f13 --- /dev/null +++ b/test_native.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +export RPC= +if [ -z $RPC ]; then + # You must set an RPC endpoint + exit 1 +fi +mkdir -p witnesses + +# Must match the values in prove_stdio.sh or build is dirty. +export RAYON_NUM_THREADS=4 +export TOKIO_WORKER_THREADS=4 +export RUST_BACKTRACE=full +export RUST_LOG=info +export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' +export RUST_MIN_STACK=33554432 + + + +MAINNETBLOCKS=" +20548415 +20240058 +19665756 +20634472 +19807080 +20634403 +19096840 +19240700 +" + +CANCUN=19426587 +TIP=`cast block-number --rpc-url $RPC` +STATICTIP=20721266 +NUMRANDOMBLOCKS=10 +RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` + +GITHASH=`git rev-parse --short HEAD` +echo "Testing against mainnet, current revision: $GITHASH." + +#BLOCKS="$MAINNETBLOCKS $RANDOMBLOCKS" +BLOCKS="$MAINNETBLOCKS" +echo "Testing blocks: $BLOCKS" + +echo "Downloading witnesses.." + +for BLOCK in $BLOCKS; do + WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" + until [ -f $WITNESS -a -s $WITNESS ]; do + echo "Fetching block $BLOCK" + cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + EXITCODE=$? + + if [ -n $EXITCODE -a -f $WITNESS -a -s $WITNESS ] + then + printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt + break + fi + + printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure| tee -a witnesses/native_results.txt + done + + echo "Witness for block $BLOCK ($WITNESS) prepared." +done + +echo "Finished downloading witnesses." +echo "Testing prepared witnesses.." + +for WITNESS in witnesses/*.native.$GITHASH.witness.json; do + echo "Testing $WITNESS" + zero_bin/tools/prove_stdio.sh $WITNESS test_only + EXITCODE=$? + if [ -n $EXITCODE ] + then + RESULT="success" + else + RESULT="failure" + fi + printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt +done + +echo "Finished testing witnesses." diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 64c0dc2ee..243bacf29 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -1,3 +1,4 @@ +use core::option::Option::None; use std::{ cmp, collections::{BTreeMap, BTreeSet, HashMap}, @@ -10,6 +11,7 @@ use anyhow::{anyhow, bail, ensure, Context as _}; use ethereum_types::{Address, U256}; use evm_arithmetization::{ generation::{mpt::AccountRlp, TrieInputs}, + jumpdest::JumpDestTableWitness, proof::TrieRoots, testing_utils::{BEACON_ROOTS_CONTRACT_ADDRESS, HISTORY_BUFFER_LENGTH}, GenerationInputs, @@ -87,6 +89,7 @@ pub fn entrypoint( }, after, withdrawals, + jumpdest_tables, }| GenerationInputs:: { txn_number_before: first_txn_ix.into(), gas_used_before: running_gas_used.into(), @@ -113,6 +116,7 @@ pub fn entrypoint( block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), burn_addr, + jumpdest_tables, }, ) .collect()) @@ -256,6 +260,8 @@ struct Batch { /// Empty for all but the final batch pub withdrawals: Vec<(Address, U256)>, + + pub jumpdest_tables: Vec>, } /// [`evm_arithmetization::generation::TrieInputs`], @@ -336,6 +342,8 @@ fn middle( )?; } + let mut jumpdest_tables = vec![]; + for txn in batch { let do_increment_txn_ix = txn.is_some(); let TxnInfo { @@ -345,6 +353,7 @@ fn middle( byte_code, new_receipt_trie_node_byte, gas_used: txn_gas_used, + jumpdest_table, }, } = txn.unwrap_or_default(); @@ -475,6 +484,8 @@ fn middle( // the transaction calling them reverted. } + jumpdest_tables.push(jumpdest_table); + if do_increment_txn_ix { txn_ix += 1; } @@ -527,6 +538,7 @@ fn middle( transactions_root: transaction_trie.root(), receipts_root: receipt_trie.root(), }, + jumpdest_tables, }); } // batch in batches diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index 901fef89a..5c72d6b3f 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -5,8 +5,11 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use ethereum_types::{Address, U256}; -use evm_arithmetization::proof::{BlockHashes, BlockMetadata}; -use keccak_hash::H256; +use evm_arithmetization::{ + jumpdest::JumpDestTableWitness, + proof::{BlockHashes, BlockMetadata}, +}; +use keccak_hash::{keccak, H256}; use mpt_trie::partial_trie::HashedPartialTrie; use plonky2::hash::hash_types::NUM_HASH_OUT_ELTS; use serde::{Deserialize, Serialize}; @@ -113,6 +116,9 @@ pub struct TxnMeta { /// Gas used by this txn (Note: not cumulative gas used). pub gas_used: u64, + + /// JumpDest table + pub jumpdest_table: Option, } /// A "trace" specific to an account for a txn. @@ -164,6 +170,16 @@ pub enum ContractCodeUsage { /// contract code will not appear in the [`BlockTrace`] map. Write(#[serde(with = "crate::hex")] Vec), } +// Question: Why has this has been removed upstream. Proably unused. +impl ContractCodeUsage { + /// Get code hash from a read or write operation of contract code. + pub fn get_code_hash(&self) -> H256 { + match self { + ContractCodeUsage::Read(hash) => *hash, + ContractCodeUsage::Write(bytes) => keccak(bytes), + } + } +} /// Other data that is needed for proof gen. #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/trace_decoder/src/jumpdest.rs b/trace_decoder/src/jumpdest.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/trace_decoder/src/jumpdest.rs @@ -0,0 +1 @@ + diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index fba82d8f7..75146fe23 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -62,6 +62,7 @@ use plonky2::field::goldilocks_field::GoldilocksField; mod type1; // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 // add backend/prod support for type 2 +mod jumpdest; #[cfg(test)] #[allow(dead_code)] mod type2; diff --git a/zero_bin/prover/src/cli.rs b/zero_bin/prover/src/cli.rs index 694194ad9..ba3147f5f 100644 --- a/zero_bin/prover/src/cli.rs +++ b/zero_bin/prover/src/cli.rs @@ -18,7 +18,7 @@ pub struct CliProverConfig { #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 19)] max_cpu_len_log: usize, /// Number of transactions in a batch to process at once. - #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 10)] + #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 1)] batch_size: usize, /// If true, save the public inputs to disk on error. #[arg(short='i', long, help_heading = HELP_HEADING, default_value_t = false)] diff --git a/zero_bin/rpc/Cargo.toml b/zero_bin/rpc/Cargo.toml index c694335d4..0215090eb 100644 --- a/zero_bin/rpc/Cargo.toml +++ b/zero_bin/rpc/Cargo.toml @@ -35,6 +35,9 @@ prover = { workspace = true } trace_decoder = { workspace = true } zero_bin_common = { workspace = true } zk_evm_common = { workspace = true } +alloy-serde = { workspace = true } +alloy-primitives = { workspace = true } +keccak-hash = { workspace = true } [build-dependencies] anyhow = { workspace = true } diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 00d56cf48..3a9f7f866 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -1,12 +1,29 @@ -use alloy::{providers::Provider, rpc::types::eth::BlockId, transports::Transport}; +use std::collections::BTreeMap; +use std::ops::Deref as _; + +use alloy::providers::ext::DebugApi; +use alloy::rpc::types::trace::geth::StructLog; +use alloy::{ + providers::Provider, + rpc::types::{eth::BlockId, trace::geth::GethTrace, Block, BlockTransactionsKind, Transaction}, + transports::Transport, +}; +use alloy_primitives::Address; use anyhow::Context as _; +use evm_arithmetization::jumpdest::JumpDestTableWitness; +use futures::stream::FuturesOrdered; +use futures::StreamExt as _; use prover::BlockProverInput; use serde::Deserialize; use serde_json::json; -use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo}; +use trace_decoder::{ + BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, +}; use zero_bin_common::provider::CachedProvider; use super::fetch_other_block_data; +use crate::jumpdest; +use crate::jumpdest::structlogprime::try_reserialize; /// Transaction traces retrieved from Erigon zeroTracer. #[derive(Debug, Deserialize)] @@ -33,16 +50,63 @@ where "debug_traceBlockByNumber".into(), (target_block_id, json!({"tracer": "zeroTracer"})), ) - .await?; + .await? + .into_iter() + .map(|ztr| ztr.result) + .collect::>(); // Grab block witness info (packed as combined trie pre-images) - let block_witness = cached_provider .get_provider() .await? .raw_request::<_, String>("eth_getWitness".into(), vec![target_block_id]) .await?; + let block = cached_provider + .get_block(target_block_id, BlockTransactionsKind::Full) + .await?; + + let tx_traces: Vec<&BTreeMap<__compat_primitive_types::H160, TxnTrace>> = tx_results + .iter() + .map(|TxnInfo { traces, meta: _ }| traces) + .collect::>(); + + let jdts: Vec> = process_transactions( + &block, + cached_provider.get_provider().await?.deref(), + &tx_traces, + ) + .await?; + + // weave in the JDTs + let txn_info = tx_results + .into_iter() + .zip(jdts) + .map( + |( + TxnInfo { + traces, + meta: + TxnMeta { + byte_code, + new_receipt_trie_node_byte, + gas_used, + jumpdest_table: _, + }, + }, + jdt, + )| TxnInfo { + traces, + meta: TxnMeta { + byte_code, + new_receipt_trie_node_byte, + gas_used, + jumpdest_table: jdt, + }, + }, + ) + .collect(); + let other_data = fetch_other_block_data(cached_provider, target_block_id, checkpoint_block_number).await?; @@ -53,9 +117,69 @@ where compact: hex::decode(block_witness.strip_prefix("0x").unwrap_or(&block_witness)) .context("invalid hex returned from call to eth_getWitness")?, }), - txn_info: tx_results.into_iter().map(|it| it.result).collect(), + txn_info, code_db: Default::default(), }, other_data, }) } + +/// Processes the transactions in the given block and updates the code db. +pub async fn process_transactions( + block: &Block, + provider: &ProviderT, + tx_traces: &[&BTreeMap<__compat_primitive_types::H160, TxnTrace>], +) -> anyhow::Result>> +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let futures = block + .transactions + .as_transactions() + .context("No transactions in block")? + .iter() + .zip(tx_traces) + .map(|(tx, &tx_trace)| process_transaction(provider, tx, tx_trace)) + .collect::>(); + + let vec_of_res = futures.collect::>().await; + vec_of_res.into_iter().collect::, _>>() +} + +/// Processes the transaction with the given transaction hash and updates the +/// accounts state. +pub async fn process_transaction( + provider: &ProviderT, + tx: &Transaction, + tx_trace: &BTreeMap<__compat_primitive_types::H160, TxnTrace>, +) -> anyhow::Result> +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let structlog_trace = provider + .debug_trace_transaction(tx.hash, jumpdest::structlog_tracing_options()) + .await?; + + let struct_logs_opt: Option> = match structlog_trace { + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), + GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) + .ok() + .map(|s| s.struct_logs), + _ => None, + }; + + let tx_traces = tx_trace + .iter() + .map(|(h, t)| (Address::from(h.to_fixed_bytes()), t.clone())) + .collect(); + + let jumpdest_table: Option = struct_logs_opt.and_then(|struct_log| { + jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces) + .map(Some) + .unwrap_or_default() + }); + + Ok(jumpdest_table) +} diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs new file mode 100644 index 000000000..7e9bde277 --- /dev/null +++ b/zero_bin/rpc/src/jumpdest.rs @@ -0,0 +1,417 @@ +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::collections::HashSet; +use std::ops::Not as _; +use std::sync::OnceLock; + +use __compat_primitive_types::H256; +use alloy::primitives::Address; +use alloy::primitives::U160; +use alloy::rpc::types::eth::Transaction; +use alloy::rpc::types::trace::geth::StructLog; +use alloy::rpc::types::trace::geth::{GethDebugTracingOptions, GethDefaultTracingOptions}; +use alloy_primitives::U256; +use anyhow::ensure; +use evm_arithmetization::jumpdest::JumpDestTableWitness; +use keccak_hash::keccak; +use trace_decoder::TxnTrace; +use tracing::trace; + +/// Tracing options for the `debug_traceTransaction` call to get structlog. +/// Used for filling JUMPDEST table. +pub(crate) fn structlog_tracing_options() -> GethDebugTracingOptions { + GethDebugTracingOptions { + config: GethDefaultTracingOptions { + disable_stack: Some(false), + // needed for CREATE2 + disable_memory: Some(false), + disable_storage: Some(true), + ..GethDefaultTracingOptions::default() + }, + tracer: None, + ..GethDebugTracingOptions::default() + } +} + +/// Provides a way to check in constant time if an address points to a +/// precompile. +fn precompiles() -> &'static HashSet
{ + static PRECOMPILES: OnceLock> = OnceLock::new(); + PRECOMPILES.get_or_init(|| { + HashSet::
::from_iter((1..=0xa).map(|x| Address::from(U160::from(x)))) + }) +} + +/// Generate at JUMPDEST table by simulating the call stack in EVM, +/// using a Geth structlog as input. +pub(crate) fn generate_jumpdest_table( + tx: &Transaction, + struct_log: &[StructLog], + tx_traces: &BTreeMap, +) -> anyhow::Result { + trace!("Generating JUMPDEST table for tx: {}", tx.hash); + ensure!(struct_log.is_empty().not(), "Structlog is empty."); + + let mut jumpdest_table = JumpDestTableWitness::default(); + + let callee_addr_to_code_hash: HashMap = tx_traces + .iter() + .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) + .filter(|(_callee_addr, code_usage)| code_usage.is_some()) + .map(|(callee_addr, code_usage)| { + (*callee_addr, code_usage.as_ref().unwrap().get_code_hash()) + }) + .collect(); + + trace!( + "Transaction: {} is a {}.", + tx.hash, + if tx.to.is_some() { + "message call" + } else { + "contract creation" + } + ); + + let entrypoint_code_hash: H256 = if let Some(to_address) = tx.to { + // Guard against transactions to a non-contract address. + ensure!( + callee_addr_to_code_hash.contains_key(&to_address), + format!("Callee addr {} is not at contract address", to_address) + ); + callee_addr_to_code_hash[&to_address] + } else { + let init = tx.input.clone(); + keccak(init) + }; + + // `None` encodes that previous `entry`` was not a JUMP or JUMPI with true + // condition, `Some(jump_target)` encodes we came from a JUMP or JUMPI with + // true condition and target `jump_target`. + let mut prev_jump = None; + + // Call depth of the previous `entry`. We initialize to 0 as this compares + // smaller to 1. + //let mut prev_depth = 0; + // The next available context. Starts at 1. Never decrements. + let mut next_ctx_available = 1; + // Immediately use context 1; + let mut call_stack = vec![(entrypoint_code_hash, next_ctx_available)]; + next_ctx_available += 1; + + for (step, entry) in struct_log.iter().enumerate() { + let op = entry.op.as_str(); + let curr_depth: usize = entry.depth.try_into().unwrap(); + + ensure!(curr_depth <= next_ctx_available, "Structlog is malformed."); + + // ensure!(call_stack.is_empty().not(), "Call stack was empty."); + while curr_depth < call_stack.len() { + call_stack.pop(); + } + + let (code_hash, ctx) = call_stack.last().unwrap(); + + trace!("TX: {:?}", tx.hash); + trace!("STEP: {:?}", step); + trace!("STEPS: {:?}", struct_log.len()); + trace!("OPCODE: {}", entry.op.as_str()); + trace!("CODE: {:?}", code_hash); + trace!("CTX: {:?}", ctx); + trace!("CURR_DEPTH: {:?}", curr_depth); + trace!("{:#?}\n", entry); + + match op { + "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 2; // actually 6 or 7. + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 + let [_gas, address, ..] = evm_stack[..] else { + unreachable!() + }; + + let callee_address = { + // Clear the upper half of the operand. + let callee_raw = *address; + // let (callee_raw, _overflow) = callee_raw.overflowing_shl(128); + // let (callee_raw, _overflow) = callee_raw.overflowing_shr(128); + + ensure!(callee_raw <= U256::from(U160::MAX)); + let lower_20_bytes = U160::from(callee_raw); + Address::from(lower_20_bytes) + }; + + if precompiles().contains(&callee_address) { + trace!("Called precompile at address {}.", &callee_address); + } else if callee_addr_to_code_hash.contains_key(&callee_address) { + let code_hash = callee_addr_to_code_hash[&callee_address]; + call_stack.push((code_hash, next_ctx_available)); + } else { + // This case happens if calling an EOA. This is described + // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun + trace!( + "Callee address {} has no associated `code_hash`.", + &callee_address + ); + } + next_ctx_available += 1; + prev_jump = None; + } + "CREATE" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 3; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [_value, _offset, _size, _salt, ..] = evm_stack[..] else { + unreachable!() + }; + + let contract_address = tx.from.create(tx.nonce); + let code_hash = callee_addr_to_code_hash[&contract_address]; + call_stack.push((code_hash, next_ctx_available)); + + next_ctx_available += 1; + prev_jump = None; + } + "CREATE2" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 4; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [_value, offset, size, salt, ..] = evm_stack[..] else { + unreachable!() + }; + ensure!(*offset < U256::from(usize::MAX)); + let offset: usize = offset.to(); + ensure!(*size < U256::from(usize::MAX)); + let size: usize = size.to(); + let salt: [u8; 32] = salt.to_be_bytes(); + + ensure!( + size == 0 + || (entry.memory.is_some() && entry.memory.as_ref().unwrap().len() >= size), + "No or insufficient memory available for {op}." + ); + let memory_raw: &[String] = entry.memory.as_ref().unwrap(); + let memory: Vec = memory_raw + .iter() + .flat_map(|s| { + let c = s.parse(); + // ensure!(c.is_ok(), "No memory."); + let a: U256 = c.unwrap(); + let d: [u8; 32] = a.to_be_bytes(); + d + }) + .collect(); + let init_code = &memory[offset..offset + size]; + let contract_address = tx.from.create2_from_code(salt, init_code); + let code_hash = callee_addr_to_code_hash[&contract_address]; + call_stack.push((code_hash, next_ctx_available)); + + next_ctx_available += 1; + prev_jump = None; + } + "JUMP" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 1; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [counter, ..] = evm_stack[..] else { + unreachable!() + }; + let jump_target = counter.to::(); + + prev_jump = Some(jump_target); + } + "JUMPI" => { + ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); + // We reverse the stack, so the order matches our assembly code. + let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); + let operands = 2; + ensure!( + evm_stack.len() >= operands, + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + evm_stack.len() + ); + let [pc, condition, ..] = evm_stack[..] else { + unreachable!() + }; + let jump_target = pc.to::(); + let jump_condition = condition.is_zero().not(); + + prev_jump = if jump_condition { + Some(jump_target) + } else { + None + }; + } + "JUMPDEST" => { + let jumped_here = if let Some(jmp_target) = prev_jump { + jmp_target == entry.pc + } else { + false + }; + let jumpdest_offset = entry.pc as usize; + if jumped_here { + jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset); + } + // else: we do not care about JUMPDESTs reached through fall-through. + prev_jump = None; + } + "EXTCODECOPY" | "EXTCODESIZE" => { + next_ctx_available += 1; + prev_jump = None; + } + // "RETURN" | "REVERT" | "STOP" => { + // ensure!(call_stack.is_empty().not(), "Call stack was empty at {op}."); + // do_pop = true; + // prev_jump = None; + // } + // "SELFDESTRUCT" => { + // do_pop = true; + // prev_jump = None; + // } + _ => { + prev_jump = None; + } + } + } + Ok(jumpdest_table) +} + +pub mod structlogprime { + use core::option::Option::None; + use std::collections::BTreeMap; + + use alloy::rpc::types::trace::geth::DefaultFrame; + use alloy_primitives::{Bytes, B256, U256}; + use serde::{ser::SerializeMap as _, Deserialize, Serialize, Serializer}; + use serde_json::Value; + + /// Geth Default struct log trace frame + /// + /// + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub(crate) struct DefaultFramePrime { + /// Whether the transaction failed + pub failed: bool, + /// How much gas was used. + pub gas: u64, + /// Output of the transaction + #[serde(serialize_with = "alloy_serde::serialize_hex_string_no_prefix")] + pub return_value: Bytes, + /// Recorded traces of the transaction + pub struct_logs: Vec, + } + + /// Represents a struct log entry in a trace + /// + /// + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] + pub(crate) struct StructLogPrime { + /// Program counter + pub pc: u64, + /// Opcode to be executed + pub op: String, + /// Remaining gas + pub gas: u64, + /// Cost for executing op + #[serde(rename = "gasCost")] + pub gas_cost: u64, + /// Current call depth + pub depth: u64, + /// Error message if any + #[serde(default, skip)] + pub error: Option, + /// EVM stack + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stack: Option>, + /// Last call's return data. Enabled via enableReturnData + #[serde( + default, + rename = "returnData", + skip_serializing_if = "Option::is_none" + )] + pub return_data: Option, + /// ref + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory: Option>, + /// Size of memory. + #[serde(default, rename = "memSize", skip_serializing_if = "Option::is_none")] + pub memory_size: Option, + /// Storage slots of current contract read from and written to. Only + /// emitted for SLOAD and SSTORE. Disabled via disableStorage + #[serde( + default, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_string_storage_map_opt" + )] + pub storage: Option>, + /// Refund counter + #[serde(default, rename = "refund", skip_serializing_if = "Option::is_none")] + pub refund_counter: Option, + } + + /// Serializes a storage map as a list of key-value pairs _without_ + /// 0x-prefix + pub(crate) fn serialize_string_storage_map_opt( + storage: &Option>, + s: S, + ) -> Result { + match storage { + None => s.serialize_none(), + Some(storage) => { + let mut m = s.serialize_map(Some(storage.len()))?; + for (key, val) in storage.iter() { + let key = format!("{:?}", key); + let val = format!("{:?}", val); + // skip the 0x prefix + m.serialize_entry(&key.as_str()[2..], &val.as_str()[2..])?; + } + m.end() + } + } + } + + impl TryInto for DefaultFramePrime { + fn try_into(self) -> Result { + let a = serde_json::to_string(&self)?; + let b: DefaultFramePrime = serde_json::from_str(&a)?; + let c = serde_json::to_string(&b)?; + let d: DefaultFrame = serde_json::from_str(&c)?; + Ok(d) + } + + type Error = anyhow::Error; + } + + pub fn try_reserialize(structlog_object: Value) -> anyhow::Result { + let a = serde_json::to_string(&structlog_object)?; + let b: DefaultFramePrime = serde_json::from_str(&a)?; + let d: DefaultFrame = b.try_into()?; + Ok(d) + } +} diff --git a/zero_bin/rpc/src/lib.rs b/zero_bin/rpc/src/lib.rs index 87a581a14..d54d1b88d 100644 --- a/zero_bin/rpc/src/lib.rs +++ b/zero_bin/rpc/src/lib.rs @@ -21,6 +21,7 @@ use trace_decoder::{BlockLevelData, OtherBlockData}; use tracing::warn; pub mod jerigon; +pub mod jumpdest; pub mod native; pub mod retry; diff --git a/zero_bin/rpc/src/main.rs b/zero_bin/rpc/src/main.rs index 12594877a..e9ddd0122 100644 --- a/zero_bin/rpc/src/main.rs +++ b/zero_bin/rpc/src/main.rs @@ -217,6 +217,9 @@ async fn main() -> anyhow::Result<()> { tracing_subscriber::Registry::default() .with( tracing_subscriber::fmt::layer() + // With the default configuration trace information is written + // to stdout, but we already use to write our payload (the witness). + .with_writer(std::io::stderr) .with_ansi(false) .compact() .with_filter(EnvFilter::from_default_env()), diff --git a/zero_bin/rpc/src/native/mod.rs b/zero_bin/rpc/src/native/mod.rs index 2e9527274..f128e6d6c 100644 --- a/zero_bin/rpc/src/native/mod.rs +++ b/zero_bin/rpc/src/native/mod.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeSet; use std::ops::Deref; use std::sync::Arc; @@ -15,7 +14,8 @@ use zero_bin_common::provider::CachedProvider; mod state; mod txn; -type CodeDb = BTreeSet>; +pub use txn::process_transaction; +pub use txn::process_transactions; /// Fetches the prover input for the given BlockId. pub async fn block_prover_input( @@ -39,7 +39,7 @@ where } /// Processes the block with the given block number and returns the block trace. -async fn process_block_trace( +pub(crate) async fn process_block_trace( cached_provider: Arc>, block_number: BlockId, ) -> anyhow::Result diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 5e3be656a..10a110e83 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -9,25 +9,27 @@ use alloy::{ Provider, }, rpc::types::{ - eth::Transaction, - eth::{AccessList, Block}, + eth::{AccessList, Block, Transaction}, trace::geth::{ - AccountState, DiffMode, GethDebugBuiltInTracerType, GethTrace, PreStateConfig, - PreStateFrame, PreStateMode, + AccountState, DiffMode, GethDebugBuiltInTracerType, GethDebugTracerType, + GethDebugTracingOptions, GethTrace, PreStateConfig, PreStateFrame, PreStateMode, + StructLog, }, - trace::geth::{GethDebugTracerType, GethDebugTracingOptions}, }, transports::Transport, }; use anyhow::Context as _; +use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; -use super::CodeDb; -use crate::Compat; +use crate::{ + jumpdest::{self, structlogprime::try_reserialize}, + Compat, +}; /// Processes the transactions in the given block and updates the code db. -pub(super) async fn process_transactions( +pub async fn process_transactions( block: &Block, provider: &ProviderT, ) -> anyhow::Result<(CodeDb, Vec)> @@ -55,7 +57,7 @@ where /// Processes the transaction with the given transaction hash and updates the /// accounts state. -async fn process_transaction( +pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, ) -> anyhow::Result<(CodeDb, TxnInfo)> @@ -63,17 +65,12 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let (tx_receipt, pre_trace, diff_trace) = fetch_tx_data(provider, &tx.hash).await?; + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = + fetch_tx_data(provider, &tx.hash).await?; let tx_status = tx_receipt.status(); let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); - let tx_meta = TxnMeta { - byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), - new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), - gas_used: tx_receipt.gas_used as u64, - }; - let (code_db, mut tx_traces) = match (pre_trace, diff_trace) { ( GethTrace::PreStateTracer(PreStateFrame::Default(read)), @@ -85,7 +82,30 @@ where // Handle case when transaction failed and a contract creation was reverted if !tx_status && tx_receipt.contract_address.is_some() { tx_traces.insert(tx_receipt.contract_address.unwrap(), TxnTrace::default()); - } + }; + + let struct_logs_opt: Option> = match structlog_trace { + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), + GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) + .ok() + .map(|s| s.struct_logs), + _ => None, + }; + + let jumpdest_table: Option = struct_logs_opt.and_then(|struct_logs| { + jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces) + .map(Some) + .unwrap_or_default() + }); + + // if jumpdest_table.is_some() { eprintln!("======================> 1")}; + + let tx_meta = TxnMeta { + byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), + new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), + gas_used: tx_receipt.gas_used as u64, + jumpdest_table, + }; Ok(( code_db, @@ -103,7 +123,12 @@ where async fn fetch_tx_data( provider: &ProviderT, tx_hash: &B256, -) -> anyhow::Result<(::ReceiptResponse, GethTrace, GethTrace), anyhow::Error> +) -> anyhow::Result<( + ::ReceiptResponse, + GethTrace, + GethTrace, + GethTrace, +)> where ProviderT: Provider, TransportT: Transport + Clone, @@ -111,14 +136,21 @@ where let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); + let structlog_trace_fut = + provider.debug_trace_transaction(*tx_hash, jumpdest::structlog_tracing_options()); - let (tx_receipt, pre_trace, diff_trace) = - futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = futures::try_join!( + tx_receipt_fut, + pre_trace_fut, + diff_trace_fut, + structlog_trace_fut + )?; Ok(( tx_receipt.context("Transaction receipt not found.")?, pre_trace, diff_trace, + structlog_trace, )) } diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index 815a7048d..f3d128d46 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -17,24 +17,25 @@ else num_procs=$(nproc) fi -# Force the working directory to always be the `tools/` directory. +# Force the working directory to always be the `tools/` directory. TOOLS_DIR=$(dirname $(realpath "$0")) PROOF_OUTPUT_DIR="${TOOLS_DIR}/proofs" -BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}" -echo "Block batch size: $BLOCK_BATCH_SIZE" +BATCH_SIZE=${BATCH_SIZE:=1} +echo "Batch size: $BATCH_SIZE" OUTPUT_LOG="${TOOLS_DIR}/output.log" PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" TEST_OUT_PATH="${TOOLS_DIR}/test.out" + # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs -export TOKIO_WORKER_THREADS=$num_procs +#export TOKIO_WORKER_THREADS=$num_procs -export RUST_MIN_STACK=33554432 -export RUST_BACKTRACE=full -export RUST_LOG=info +#export RUST_MIN_STACK=33554432 +#export RUST_BACKTRACE=full +#export RUST_LOG=info # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' @@ -95,7 +96,7 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --quiet --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $TEST_OUT_PATH + cargo run --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" rm $TEST_OUT_PATH @@ -108,10 +109,9 @@ fi cargo build --release --jobs "$num_procs" - start_time=$(date +%s%N) -"${TOOLS_DIR}/../../target/release/leader" --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE \ - --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $OUTPUT_LOG +"${TOOLS_DIR}/../../target/release/leader" --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE \ + --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $OUTPUT_LOG end_time=$(date +%s%N) set +o pipefail From 459148215f9e3b92d8a041ae92a1ec4eceecf296 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 15 Sep 2024 20:26:10 +0200 Subject: [PATCH 002/112] feedback + cleanups --- zero_bin/rpc/src/jumpdest.rs | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 7e9bde277..2a35cc877 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -105,11 +105,11 @@ pub(crate) fn generate_jumpdest_table( ensure!(curr_depth <= next_ctx_available, "Structlog is malformed."); - // ensure!(call_stack.is_empty().not(), "Call stack was empty."); while curr_depth < call_stack.len() { call_stack.pop(); } + ensure!(call_stack.is_empty().not(), "Call stack was empty."); let (code_hash, ctx) = call_stack.last().unwrap(); trace!("TX: {:?}", tx.hash); @@ -179,6 +179,7 @@ pub(crate) fn generate_jumpdest_table( }; let contract_address = tx.from.create(tx.nonce); + ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); let code_hash = callee_addr_to_code_hash[&contract_address]; call_stack.push((code_hash, next_ctx_available)); @@ -202,26 +203,30 @@ pub(crate) fn generate_jumpdest_table( let offset: usize = offset.to(); ensure!(*size < U256::from(usize::MAX)); let size: usize = size.to(); + let memory_size = entry.memory.as_ref().unwrap().len(); let salt: [u8; 32] = salt.to_be_bytes(); ensure!( - size == 0 - || (entry.memory.is_some() && entry.memory.as_ref().unwrap().len() >= size), - "No or insufficient memory available for {op}." + entry.memory.is_some() && size <= memory_size, + "No or insufficient memory available for {op}. Contract size is {size} while memory size is {memory_size}." ); let memory_raw: &[String] = entry.memory.as_ref().unwrap(); - let memory: Vec = memory_raw + let memory_parsed: Vec> = memory_raw .iter() - .flat_map(|s| { + .map(|s| { let c = s.parse(); - // ensure!(c.is_ok(), "No memory."); + ensure!(c.is_ok(), "Parsing memory failed."); let a: U256 = c.unwrap(); let d: [u8; 32] = a.to_be_bytes(); - d + Ok(d) }) .collect(); + let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); + let memory: Vec = mem_res?.concat(); + let init_code = &memory[offset..offset + size]; let contract_address = tx.from.create2_from_code(salt, init_code); + ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); let code_hash = callee_addr_to_code_hash[&contract_address]; call_stack.push((code_hash, next_ctx_available)); @@ -284,15 +289,6 @@ pub(crate) fn generate_jumpdest_table( next_ctx_available += 1; prev_jump = None; } - // "RETURN" | "REVERT" | "STOP" => { - // ensure!(call_stack.is_empty().not(), "Call stack was empty at {op}."); - // do_pop = true; - // prev_jump = None; - // } - // "SELFDESTRUCT" => { - // do_pop = true; - // prev_jump = None; - // } _ => { prev_jump = None; } From 91c29454e9509ad13eb5e5235e84eae967575132 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 15 Sep 2024 21:35:55 +0200 Subject: [PATCH 003/112] cleanups --- test_jerigon.sh | 9 +++++---- zero_bin/rpc/src/jerigon.rs | 2 ++ zero_bin/rpc/src/native/txn.rs | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/test_jerigon.sh b/test_jerigon.sh index 49f88ed12..fe335869b 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -60,17 +60,18 @@ do done # TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -TESTNETBLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -TESTNETBLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` +BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` SHUF=`shuf -e $TESTNETBLOCKS` echo $SHUF +echo "Testing: $BLOCKS" -for BLOCK in $TESTNETBLOCKS; do +for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` - WITNESS="$witnesses/$BLOCK.jerigon.$GITHASH.witness.json" + WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Checking block $BLOCK" diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 3a9f7f866..2fefd90ec 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -19,6 +19,7 @@ use serde_json::json; use trace_decoder::{ BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, }; +use tracing::debug; use zero_bin_common::provider::CachedProvider; use super::fetch_other_block_data; @@ -177,6 +178,7 @@ where let jumpdest_table: Option = struct_logs_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces) + .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) .map(Some) .unwrap_or_default() }); diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 10a110e83..f0be69ebf 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -22,6 +22,7 @@ use anyhow::Context as _; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; +use tracing::debug; use crate::{ jumpdest::{self, structlogprime::try_reserialize}, @@ -94,12 +95,11 @@ where let jumpdest_table: Option = struct_logs_opt.and_then(|struct_logs| { jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces) + .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) .map(Some) .unwrap_or_default() }); - // if jumpdest_table.is_some() { eprintln!("======================> 1")}; - let tx_meta = TxnMeta { byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), From b58c5d67cfd4f0e7948e6a2419318ce5e0a40f25 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sun, 15 Sep 2024 23:08:47 +0200 Subject: [PATCH 004/112] fix overflow --- zero_bin/rpc/src/jumpdest.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 2a35cc877..b4f048f28 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -199,9 +199,9 @@ pub(crate) fn generate_jumpdest_table( let [_value, offset, size, salt, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*offset < U256::from(usize::MAX)); + ensure!(*offset <= U256::from(usize::MAX)); let offset: usize = offset.to(); - ensure!(*size < U256::from(usize::MAX)); + ensure!(*size <= U256::from(usize::MAX)); let size: usize = size.to(); let memory_size = entry.memory.as_ref().unwrap().len(); let salt: [u8; 32] = salt.to_be_bytes(); @@ -246,7 +246,8 @@ pub(crate) fn generate_jumpdest_table( let [counter, ..] = evm_stack[..] else { unreachable!() }; - let jump_target = counter.to::(); + ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + let jump_target: u64 = counter.to(); prev_jump = Some(jump_target); } @@ -260,10 +261,11 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [pc, condition, ..] = evm_stack[..] else { + let [counter, condition, ..] = evm_stack[..] else { unreachable!() }; - let jump_target = pc.to::(); + ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); prev_jump = if jump_condition { From 037fb5710c5fd66888f1fe1f4a00960c5eddca03 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 10:58:36 +0200 Subject: [PATCH 005/112] fmt --- zero_bin/rpc/src/jumpdest.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index b4f048f28..7ea971e13 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -246,7 +246,10 @@ pub(crate) fn generate_jumpdest_table( let [counter, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + ensure!( + *counter <= U256::from(u64::MAX), + "Operand for {op} caused overflow." + ); let jump_target: u64 = counter.to(); prev_jump = Some(jump_target); @@ -264,7 +267,10 @@ pub(crate) fn generate_jumpdest_table( let [counter, condition, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*counter <= U256::from(u64::MAX), "Operand for {op} caused overflow."); + ensure!( + *counter <= U256::from(u64::MAX), + "Operand for {op} caused overflow." + ); let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); From 85ee8c274b44a7506c7e21732c88993c76e88fe3 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 14:44:37 +0200 Subject: [PATCH 006/112] fix testscripts --- test_jerigon.sh | 18 ++++++++++-------- test_native.sh | 8 ++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/test_jerigon.sh b/test_jerigon.sh index fe335869b..deac74b2b 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -euxo pipefail +set -uxo pipefail #export CARGO_LOG=cargo::core::compiler::fingerprint=debug export RPC= @@ -15,7 +15,7 @@ export TOKIO_WORKER_THREADS=4 export RUST_BACKTRACE=full export RUST_LOG=info export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' -export RUST_MIN_STACK=33554432 +export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` echo "Testing against jergion, current revision: $GITHASH." @@ -60,16 +60,18 @@ do done # TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +# BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" +#BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` -BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` SHUF=`shuf -e $TESTNETBLOCKS` echo $SHUF -echo "Testing: $BLOCKS" +#echo "Testing: $BLOCKS" +printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt +printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt -for BLOCK in $BLOCKS; do +for BLOCK in {1..256}; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" @@ -77,13 +79,13 @@ for BLOCK in $BLOCKS; do echo "Checking block $BLOCK" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? - if [ -n $EXITCODE ] + if [ $EXITCODE -eq 0 ] then RESULT="success" else RESULT="failure" fi - printf "%10i %s %s\n" $BLOCK $GITHASH $RESULT | tee -a result.txt + printf "%s %10i %s\n" $GITHASH $BLOCK $RESULT | tee -a witnesses/jerigon_results.txt done exit 0 diff --git a/test_native.sh b/test_native.sh index 913f27f13..fcb85fba5 100755 --- a/test_native.sh +++ b/test_native.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -euxo pipefail +set -uxo pipefail export RPC= if [ -z $RPC ]; then @@ -49,10 +49,10 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" until [ -f $WITNESS -a -s $WITNESS ]; do echo "Fetching block $BLOCK" - cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + cargo run --release --verbose --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS EXITCODE=$? - if [ -n $EXITCODE -a -f $WITNESS -a -s $WITNESS ] + if [ $EXITCODE -eq 0 -a -f $WITNESS -a -s $WITNESS ] then printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt break @@ -71,7 +71,7 @@ for WITNESS in witnesses/*.native.$GITHASH.witness.json; do echo "Testing $WITNESS" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? - if [ -n $EXITCODE ] + if [ $EXITCODE -eq 0 ] then RESULT="success" else From 1243768b0b026305d326394903bed23c5ba8b546 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 14:45:24 +0200 Subject: [PATCH 007/112] refactor --- zero_bin/rpc/src/jerigon.rs | 34 ++++++----------- zero_bin/rpc/src/jumpdest.rs | 68 +++++++++++++++++++++++++++++----- zero_bin/rpc/src/native/txn.rs | 32 +++++++--------- 3 files changed, 84 insertions(+), 50 deletions(-) diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 2fefd90ec..3c36f7454 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -1,11 +1,9 @@ use std::collections::BTreeMap; use std::ops::Deref as _; -use alloy::providers::ext::DebugApi; -use alloy::rpc::types::trace::geth::StructLog; use alloy::{ providers::Provider, - rpc::types::{eth::BlockId, trace::geth::GethTrace, Block, BlockTransactionsKind, Transaction}, + rpc::types::{eth::BlockId, Block, BlockTransactionsKind, Transaction}, transports::Transport, }; use alloy_primitives::Address; @@ -23,8 +21,7 @@ use tracing::debug; use zero_bin_common::provider::CachedProvider; use super::fetch_other_block_data; -use crate::jumpdest; -use crate::jumpdest::structlogprime::try_reserialize; +use crate::jumpdest::{self, get_normalized_structlog}; /// Transaction traces retrieved from Erigon zeroTracer. #[derive(Debug, Deserialize)] @@ -159,28 +156,21 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let structlog_trace = provider - .debug_trace_transaction(tx.hash, jumpdest::structlog_tracing_options()) - .await?; - - let struct_logs_opt: Option> = match structlog_trace { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), - GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) - .ok() - .map(|s| s.struct_logs), - _ => None, - }; - let tx_traces = tx_trace .iter() .map(|(h, t)| (Address::from(h.to_fixed_bytes()), t.clone())) .collect(); - let jumpdest_table: Option = struct_logs_opt.and_then(|struct_log| { - jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces) - .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) - .map(Some) - .unwrap_or_default() + let structlog_opt = get_normalized_structlog(provider, &tx.hash).await?; + + let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { + jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( + |error| { + debug!("JumpDestTable generation failed with reason: {}", error); + None + }, + Some, + ) }); Ok(jumpdest_table) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 7ea971e13..97531eee9 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -7,25 +7,31 @@ use std::sync::OnceLock; use __compat_primitive_types::H256; use alloy::primitives::Address; use alloy::primitives::U160; +use alloy::providers::ext::DebugApi; +use alloy::providers::Provider; use alloy::rpc::types::eth::Transaction; use alloy::rpc::types::trace::geth::StructLog; use alloy::rpc::types::trace::geth::{GethDebugTracingOptions, GethDefaultTracingOptions}; +use alloy::transports::RpcError; +use alloy::transports::Transport; +use alloy::transports::TransportErrorKind; +use alloy_primitives::B256; use alloy_primitives::U256; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use keccak_hash::keccak; +use structlogprime::normalize_structlog; use trace_decoder::TxnTrace; use tracing::trace; -/// Tracing options for the `debug_traceTransaction` call to get structlog. -/// Used for filling JUMPDEST table. -pub(crate) fn structlog_tracing_options() -> GethDebugTracingOptions { +/// Pass `true` for the components needed. +fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDebugTracingOptions { GethDebugTracingOptions { config: GethDefaultTracingOptions { - disable_stack: Some(false), + disable_stack: Some(!stack), // needed for CREATE2 - disable_memory: Some(false), - disable_storage: Some(true), + disable_memory: Some(!memory), + disable_storage: Some(!storage), ..GethDefaultTracingOptions::default() }, tracer: None, @@ -33,6 +39,38 @@ pub(crate) fn structlog_tracing_options() -> GethDebugTracingOptions { } } +fn trace_contains_create2(structlog: Vec) -> bool { + structlog.iter().any(|entry| entry.op == "CREATE2") +} + +// Gets the lightest possible structlog for transcation `tx_hash`. +pub(crate) async fn get_normalized_structlog( + provider: &ProviderT, + tx_hash: &B256, +) -> Result>, RpcError> +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let light_structlog_trace = provider + .debug_trace_transaction(*tx_hash, structlog_tracing_options(false, false, false)) + .await?; + + let structlogs_opt: Option> = normalize_structlog(light_structlog_trace).await; + + let need_memory = structlogs_opt.is_some_and(trace_contains_create2); + + let structlog = provider + .debug_trace_transaction( + *tx_hash, + structlog_tracing_options(true, need_memory, false), + ) + .await?; + + let ret = normalize_structlog(structlog).await; + Ok(ret) +} + /// Provides a way to check in constant time if an address points to a /// precompile. fn precompiles() -> &'static HashSet
{ @@ -309,7 +347,7 @@ pub mod structlogprime { use core::option::Option::None; use std::collections::BTreeMap; - use alloy::rpc::types::trace::geth::DefaultFrame; + use alloy::rpc::types::trace::geth::{DefaultFrame, GethTrace, StructLog}; use alloy_primitives::{Bytes, B256, U256}; use serde::{ser::SerializeMap as _, Deserialize, Serialize, Serializer}; use serde_json::Value; @@ -412,10 +450,22 @@ pub mod structlogprime { type Error = anyhow::Error; } - pub fn try_reserialize(structlog_object: Value) -> anyhow::Result { - let a = serde_json::to_string(&structlog_object)?; + pub fn try_reserialize(structlog_object: &Value) -> anyhow::Result { + let a = serde_json::to_string(structlog_object)?; let b: DefaultFramePrime = serde_json::from_str(&a)?; let d: DefaultFrame = b.try_into()?; Ok(d) } + + pub(crate) async fn normalize_structlog( + unnormalized_structlog: GethTrace, + ) -> Option> { + match unnormalized_structlog { + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), + GethTrace::JS(structlog_js_object) => try_reserialize(&structlog_js_object) + .ok() + .map(|s| s.struct_logs), + _ => None, + } + } } diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index f0be69ebf..80d2f367b 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -18,14 +18,14 @@ use alloy::{ }, transports::Transport, }; -use anyhow::Context as _; +use anyhow::{Context as _, Ok}; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use tracing::debug; use crate::{ - jumpdest::{self, structlogprime::try_reserialize}, + jumpdest::{self, get_normalized_structlog}, Compat, }; @@ -66,7 +66,7 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let (tx_receipt, pre_trace, diff_trace, structlog_trace) = + let (tx_receipt, pre_trace, diff_trace, structlog_opt) = fetch_tx_data(provider, &tx.hash).await?; let tx_status = tx_receipt.status(); let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); @@ -85,19 +85,14 @@ where tx_traces.insert(tx_receipt.contract_address.unwrap(), TxnTrace::default()); }; - let struct_logs_opt: Option> = match structlog_trace { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), - GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) - .ok() - .map(|s| s.struct_logs), - _ => None, - }; - - let jumpdest_table: Option = struct_logs_opt.and_then(|struct_logs| { - jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces) - .map_err(|error| debug!("JumpDestTable generation failed with reason: {}", error)) - .map(Some) - .unwrap_or_default() + let jumpdest_table: Option = structlog_opt.and_then(|struct_logs| { + jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces).map_or_else( + |error| { + debug!("JumpDestTable generation failed with reason: {}", error); + None + }, + Some, + ) }); let tx_meta = TxnMeta { @@ -127,7 +122,7 @@ async fn fetch_tx_data( ::ReceiptResponse, GethTrace, GethTrace, - GethTrace, + Option>, )> where ProviderT: Provider, @@ -136,8 +131,7 @@ where let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); - let structlog_trace_fut = - provider.debug_trace_transaction(*tx_hash, jumpdest::structlog_tracing_options()); + let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); let (tx_receipt, pre_trace, diff_trace, structlog_trace) = futures::try_join!( tx_receipt_fut, From e7244c6bf648343800572a10024d91050cfd5cd7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 16 Sep 2024 22:29:01 +0200 Subject: [PATCH 008/112] for testing --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 10 ++++++++-- test_jerigon.sh | 9 ++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index d0cb867fa..19b1db7c8 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -11,7 +11,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use anyhow::anyhow; use ethereum_types::{BigEndianHash, U256}; use keccak_hash::H256; -use log::Level; +use log::{trace, Level}; use mpt_trie::partial_trie::PartialTrie; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -167,7 +167,13 @@ pub(crate) fn set_jumpdest_analysis_inputs_rpc( let ctx_proofs = (*jumpdest_table_rpc) .iter() .flat_map(|(code_addr, ctx_jumpdests)| { - prove_context_jumpdests(&code_map[code_addr], ctx_jumpdests) + let code = if code_map.contains_key(code_addr) { + &code_map[code_addr] + } else { + &vec![] + }; + trace!("code: {:?} <============", &code); + prove_context_jumpdests(code, ctx_jumpdests) }) .collect(); JumpDestTableProcessed::new(ctx_proofs) diff --git a/test_jerigon.sh b/test_jerigon.sh index deac74b2b..0bd16b311 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -41,6 +41,13 @@ TESTNETBLOCKS=" KNOWNFAILED=" +2 +15 +28 +35 +37 +43 +65 28 444 " @@ -71,7 +78,7 @@ echo $SHUF printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt -for BLOCK in {1..256}; do +for BLOCK in {66..688}; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" From 16e9c261ed14dcb9ec666863635f61deb6e1716d Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 17 Sep 2024 22:24:29 +0200 Subject: [PATCH 009/112] extract initcode --- .../benches/fibonacci_25m_gas.rs | 2 +- .../src/cpu/kernel/interpreter.rs | 7 +++- .../src/cpu/kernel/tests/add11.rs | 4 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 2 +- evm_arithmetization/src/generation/mod.rs | 4 +- .../src/generation/prover_input.rs | 21 +++++++--- evm_arithmetization/tests/add11_yml.rs | 2 +- evm_arithmetization/tests/erc20.rs | 2 +- evm_arithmetization/tests/erc721.rs | 2 +- evm_arithmetization/tests/log_opcode.rs | 2 +- evm_arithmetization/tests/selfdestruct.rs | 2 +- evm_arithmetization/tests/simple_transfer.rs | 2 +- evm_arithmetization/tests/withdrawals.rs | 2 +- test_jerigon.sh | 27 +++++++++++-- trace_decoder/src/core.rs | 39 +++++++++++++++---- 15 files changed, 90 insertions(+), 30 deletions(-) diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index 57b902c63..2b35b2a1c 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -193,7 +193,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 19b1db7c8..7bc5e1dcc 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -172,7 +172,12 @@ pub(crate) fn set_jumpdest_analysis_inputs_rpc( } else { &vec![] }; - trace!("code: {:?} <============", &code); + trace!( + "code: {:?}, code_addr: {:?} <============", + &code, + &code_addr + ); + trace!("code_map: {:?}", &code_map); prove_context_jumpdests(code, ctx_jumpdests) }) .collect(); diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index 1cc5588e1..fbae6647d 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -194,7 +194,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let initial_stack = vec![]; @@ -372,7 +372,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index 08fe57b1a..673b516b0 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,7 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 87d9ed1f7..e5ec7bde3 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -111,7 +111,7 @@ pub struct GenerationInputs { /// A table listing each JUMPDESTs reached in each call context under /// associated code hash. - pub jumpdest_tables: Vec>, + pub jumpdest_tables: Option, } /// A lighter version of [`GenerationInputs`], which have been trimmed @@ -165,7 +165,7 @@ pub struct TrimmedGenerationInputs { /// A list of tables listing each JUMPDESTs reached in each call context /// under associated code hash. - pub jumpdest_tables: Vec>, + pub jumpdest_tables: Option, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index ae49366b3..dc59bda2c 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -7,6 +7,7 @@ use anyhow::{bail, Error, Result}; use ethereum_types::{BigEndianHash, H256, U256, U512}; use itertools::Itertools; use keccak_hash::keccak; +use log::trace; use num_bigint::BigUint; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -769,15 +770,18 @@ impl GenerationState { dbg!(&self.inputs.jumpdest_tables); eprintln!("Generating JUMPDEST tables"); // w for witness - let txn_idx = self.next_txn_index - 1; - let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref(); - let rpc = rpcw.map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + // let txn_idx = self.next_txn_index - 1; + // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code + let rpcw = &self.inputs.jumpdest_tables; + let rpc = rpcw + .as_ref() + .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); if let Some((_sim, simw)) = simulate_cpu_and_get_user_jumps("terminate_common", self) { - if rpcw.is_some() && rpcw.unwrap() != &simw.clone() { + if rpcw.is_some() && rpcw.clone().unwrap() != simw.clone() { println!("SIMW {}", simw.clone()); - println!("RPCW {}", rpcw.unwrap()); - assert_eq!(simw.clone(), *rpcw.unwrap()); + println!("RPCW {}", rpcw.clone().unwrap()); + assert_eq!(simw.clone(), rpcw.clone().unwrap()); } } self.jumpdest_table = rpc; @@ -796,6 +800,11 @@ impl GenerationState { self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); + trace!( + "ctx: {ctx}, code_hash: {:?} code: {:?}", + keccak(code.clone()), + code + ); for offset in jumpdest_table.clone() { jdtw.insert(keccak(code.clone()), ctx, offset); } diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 51ac662df..12a84fbe7 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,7 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index f13f541b4..475d03a8e 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -196,7 +196,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index 6386dbe69..8d30f85d1 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -200,7 +200,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index d9fe6f234..951ebbb28 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -267,7 +267,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 27d68af7b..221113b26 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -171,7 +171,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index 4109ae09c..de8e096cf 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -163,7 +163,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index 3dfe13e43..2aa776e54 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -106,7 +106,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: vec![], + jumpdest_tables: Default::default(), }; let max_cpu_len_log = 20; diff --git a/test_jerigon.sh b/test_jerigon.sh index 0bd16b311..b1021b03b 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -48,10 +48,31 @@ KNOWNFAILED=" 37 43 65 + 28 444 -" +43 +460 +461 +462 +463 +464 +465 +467 +468 +474 +475 +476 +566 +662 +664 +665 +667 +670 +72 +77 +" # 470..663 from Robin @@ -78,11 +99,11 @@ echo $SHUF printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt -for BLOCK in {66..688}; do +for BLOCK in $KNOWNFAILED; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" - cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + timeout 2m cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Checking block $BLOCK" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 243bacf29..2cfb934ed 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -1,4 +1,4 @@ -use core::option::Option::None; +use core::{convert::Into as _, option::Option::None}; use std::{ cmp, collections::{BTreeMap, BTreeSet, HashMap}, @@ -6,6 +6,11 @@ use std::{ }; use alloy::primitives::address; +use alloy::{ + consensus::{Transaction, TxEnvelope}, + primitives::TxKind, + rlp::Decodable as _, +}; use alloy_compat::Compat as _; use anyhow::{anyhow, bail, ensure, Context as _}; use ethereum_types::{Address, U256}; @@ -97,7 +102,7 @@ pub fn entrypoint( running_gas_used += gas_used; running_gas_used.into() }, - signed_txns: byte_code.into_iter().map(Into::into).collect(), + signed_txns: byte_code.clone().into_iter().map(Into::into).collect(), withdrawals, ger_data: None, tries: TrieInputs { @@ -109,14 +114,34 @@ pub fn entrypoint( trie_roots_after: after, checkpoint_state_trie_root, checkpoint_consolidated_hash, - contract_code: contract_code - .into_iter() - .map(|it| (keccak_hash::keccak(&it), it)) - .collect(), + contract_code: { + let cc = contract_code.into_iter(); + + let initcodes = + byte_code + .iter() + .filter_map(|nonempty_txn_raw| -> Option> { + let tx_envelope = + TxEnvelope::decode(&mut &nonempty_txn_raw[..]).unwrap(); + match tx_envelope.to() { + TxKind::Create => Some(tx_envelope.input().to_vec()), + TxKind::Call(_address) => None, + } + }); + + cc.chain(initcodes) + .map(|it| (keccak_hash::keccak(&it), it)) + .collect() + }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), burn_addr, - jumpdest_tables, + jumpdest_tables: { + jumpdest_tables + .into_iter() + .collect::>>() + .map(|vj| JumpDestTableWitness::merge(vj.iter()).0) + }, }, ) .collect()) From f3871d96717b601852102384f013cae6f683964e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 17 Sep 2024 22:47:49 +0200 Subject: [PATCH 010/112] improve test script --- test_jerigon.sh | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/test_jerigon.sh b/test_jerigon.sh index b1021b03b..53309228c 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -uxo pipefail +set -uo pipefail #export CARGO_LOG=cargo::core::compiler::fingerprint=debug export RPC= @@ -20,7 +20,7 @@ export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` echo "Testing against jergion, current revision: $GITHASH." -TESTNETBLOCKS=" +CIBLOCKS=" 1 2 3 @@ -81,30 +81,28 @@ do ROBIN+=" $i" done -# Pick random blocks -for i in {1..10} -do - RANDOMBLOCKS+=" $((1 + $RANDOM % 688))" -done - -# TESTNETBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -# BLOCKS="$ROBIN $RANDOMBLOCKS $TESTNETBLOCKS" -#BLOCKS=`echo $TESTNETBLOCKS | sed 's/\s/\n/g'` - -SHUF=`shuf -e $TESTNETBLOCKS` -echo $SHUF +TIP=688 +NUMRANDOMBLOCKS=10 +RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` +# CIBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $CIBLOCKS" +#BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" +#BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` +BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" +BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt -printf "---------------------------\n" | tee -a witnesses/jerigon_results.txt +echo "---------------------------" | tee -a witnesses/jerigon_results.txt -for BLOCK in $KNOWNFAILED; do +for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" timeout 2m cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - echo "Checking block $BLOCK" + echo "Testing blocks:" + echo $BLOCKS + echo "Now testing block $BLOCK" zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] From 4fd6b8b7028c6a9d769b20f612533499018609ff Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 10:35:20 +0200 Subject: [PATCH 011/112] fix stack issue --- zero_bin/rpc/src/jumpdest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 97531eee9..67c5e68a2 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -212,7 +212,7 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [_value, _offset, _size, _salt, ..] = evm_stack[..] else { + let [_value, _offset, _size, ..] = evm_stack[..] else { unreachable!() }; From 88eb73d7db4f49e5f440e3c7d8e875c5343d7cd8 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 13:23:07 +0200 Subject: [PATCH 012/112] random fixes --- Cargo.toml | 2 +- .../src/generation/prover_input.rs | 24 +++--- test_jerigon.sh | 77 +++++++++++++++++-- zero_bin/rpc/src/jumpdest.rs | 4 +- 4 files changed, 88 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 84a438806..294681d05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ tiny-keccak = "2.0.2" tokio = { version = "1.38.0", features = ["full"] } toml = "0.8.14" tower = "0.4" -tracing = "0.1" +tracing = { version = "0.1", features = ["attributes"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] } u4 = "0.1.0" uint = "0.9.5" diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index dc59bda2c..8751e2ded 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -767,24 +767,30 @@ impl GenerationState { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - dbg!(&self.inputs.jumpdest_tables); eprintln!("Generating JUMPDEST tables"); + dbg!(&self.inputs.jumpdest_tables); // w for witness // let txn_idx = self.next_txn_index - 1; // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code let rpcw = &self.inputs.jumpdest_tables; - let rpc = rpcw + let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); - if let Some((_sim, simw)) = simulate_cpu_and_get_user_jumps("terminate_common", self) { - if rpcw.is_some() && rpcw.clone().unwrap() != simw.clone() { - println!("SIMW {}", simw.clone()); - println!("RPCW {}", rpcw.clone().unwrap()); - assert_eq!(simw.clone(), rpcw.clone().unwrap()); - } + let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); + + let (sim, simw): (Option, Option) = + sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); + + if let (Some(rw), Some(sw)) = (rpcw, simw) + && rw != &sw + { + trace!("SIMW {}", sw); + trace!("RPCW {}", rw); + assert_eq!(rw, &sw); } - self.jumpdest_table = rpc; + + self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; Ok(()) } diff --git a/test_jerigon.sh b/test_jerigon.sh index 53309228c..2b1f5c2d5 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -40,7 +40,71 @@ CIBLOCKS=" " -KNOWNFAILED=" +STILLFAIL=" +37 +75 +15 +35 +43 +72 +77 +184 +460 +461 +462 +463 +464 +465 +467 +468 +474 +475 +476 +566 +662 +664 +665 +667 +670 +477 +478 +444 +" + +JUMPI=" +662 +664 +665 +667 +670 +" + +CONTAINSKEY=" +461 +462 +463 +464 +465 +467 +468 +474 +475 +476 +72 +" + +CREATE2=" +43 +566 +77 +" + +DECODING=" +477 +478 +" + +USEDTOFAIL=" 2 15 28 @@ -50,7 +114,6 @@ KNOWNFAILED=" 65 28 -444 43 460 @@ -88,7 +151,8 @@ RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` # CIBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` -BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" +#BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" +BLOCKS="$DECODING" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" @@ -99,10 +163,9 @@ for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" - timeout 2m cargo run --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - echo "Testing blocks:" - echo $BLOCKS - echo "Now testing block $BLOCK" + timeout 2m cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + echo "Testing blocks: $BLOCKS." + echo "Now testing block $BLOCK .." zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 67c5e68a2..e31e90bf5 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -82,13 +82,13 @@ fn precompiles() -> &'static HashSet
{ /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. +#[instrument] pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], tx_traces: &BTreeMap, ) -> anyhow::Result { trace!("Generating JUMPDEST table for tx: {}", tx.hash); - ensure!(struct_log.is_empty().not(), "Structlog is empty."); let mut jumpdest_table = JumpDestTableWitness::default(); @@ -123,7 +123,7 @@ pub(crate) fn generate_jumpdest_table( keccak(init) }; - // `None` encodes that previous `entry`` was not a JUMP or JUMPI with true + // `None` encodes that previous `entry` was not a JUMP or JUMPI with true // condition, `Some(jump_target)` encodes we came from a JUMP or JUMPI with // true condition and target `jump_target`. let mut prev_jump = None; From 39cd26c77d5f395fd3d1cf1e4c06e62b69ee2916 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 16:37:42 +0200 Subject: [PATCH 013/112] fix CREATE2 --- .../src/generation/prover_input.rs | 18 +++-- test_jerigon.sh | 2 + zero_bin/rpc/src/jerigon.rs | 13 ++-- zero_bin/rpc/src/jumpdest.rs | 66 ++++++++++++------- zero_bin/rpc/src/native/txn.rs | 2 +- zero_bin/tools/prove_stdio.sh | 4 +- 6 files changed, 65 insertions(+), 40 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 8751e2ded..1919f98d2 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -7,7 +7,7 @@ use anyhow::{bail, Error, Result}; use ethereum_types::{BigEndianHash, H256, U256, U512}; use itertools::Itertools; use keccak_hash::keccak; -use log::trace; +use log::{info, trace}; use num_bigint::BigUint; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -767,7 +767,7 @@ impl GenerationState { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - eprintln!("Generating JUMPDEST tables"); + info!("Generating JUMPDEST tables"); dbg!(&self.inputs.jumpdest_tables); // w for witness // let txn_idx = self.next_txn_index - 1; @@ -775,20 +775,18 @@ impl GenerationState { let rpcw = &self.inputs.jumpdest_tables; let rpc: Option = rpcw .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + .map(|jdt| set_jumpdest_analysis_inputs_rpc(&jdt, &self.inputs.contract_code)); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); let (sim, simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) - && rw != &sw - { - trace!("SIMW {}", sw); - trace!("RPCW {}", rw); - assert_eq!(rw, &sw); - } + if let (Some(rw), Some(sw)) = (rpcw, simw) && rw != &sw { + info!("SIMW {}", sw); + info!("RPCW {}", rw); + assert_eq!(rw, &sw); + } self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; diff --git a/test_jerigon.sh b/test_jerigon.sh index 2b1f5c2d5..5362d6156 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -163,9 +163,11 @@ for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" + export RUST_LOG=rpc=trace timeout 2m cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." + export RUST_LOG=info zero_bin/tools/prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 3c36f7454..0c3900ab8 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -3,7 +3,7 @@ use std::ops::Deref as _; use alloy::{ providers::Provider, - rpc::types::{eth::BlockId, Block, BlockTransactionsKind, Transaction}, + rpc::types::{eth::BlockId, trace::geth::StructLog, Block, BlockTransactionsKind, Transaction}, transports::Transport, }; use alloy_primitives::Address; @@ -161,15 +161,20 @@ where .map(|(h, t)| (Address::from(h.to_fixed_bytes()), t.clone())) .collect(); - let structlog_opt = get_normalized_structlog(provider, &tx.hash).await?; + let structlog_opt: Option> = get_normalized_structlog(provider, &tx.hash) + .await + .ok() + .flatten(); let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { - debug!("JumpDestTable generation failed with reason: {}", error); + debug!("{:#?}: JumpDestTable generation failed with reason: {}", tx.hash, error); None }, - Some, + |jdt|{ + debug!("{:#?}: JumpDestTable generation succeceeded with result: {}",tx.hash, jdt); + Some(jdt)}, ) }); diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index e31e90bf5..321acd9fa 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -1,3 +1,5 @@ +use core::default::Default; +use core::time::Duration; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::HashSet; @@ -21,8 +23,17 @@ use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use keccak_hash::keccak; use structlogprime::normalize_structlog; +use tokio::time::timeout; use trace_decoder::TxnTrace; -use tracing::trace; +use tracing::{instrument, trace}; + +/// The maximum time we are willing to wait for a structlog before failing over +/// to simulating the JumpDest analysis. +const TIMEOUT_LIMIT: Duration = Duration::from_secs(10); + +/// Structure of Etheruem memory +type Word = [u8; 32]; +const WORDSIZE: usize = std::mem::size_of::(); /// Pass `true` for the components needed. fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDebugTracingOptions { @@ -52,6 +63,7 @@ where ProviderT: Provider, TransportT: Transport + Clone, { + // Optimization: It may be a better default to pull the stack immediately. let light_structlog_trace = provider .debug_trace_transaction(*tx_hash, structlog_tracing_options(false, false, false)) .await?; @@ -59,16 +71,19 @@ where let structlogs_opt: Option> = normalize_structlog(light_structlog_trace).await; let need_memory = structlogs_opt.is_some_and(trace_contains_create2); + trace!("Need structlog with memory: {need_memory}"); - let structlog = provider - .debug_trace_transaction( - *tx_hash, - structlog_tracing_options(true, need_memory, false), - ) - .await?; + let structlog = provider.debug_trace_transaction( + *tx_hash, + structlog_tracing_options(true, need_memory, false), + ); - let ret = normalize_structlog(structlog).await; - Ok(ret) + match timeout(TIMEOUT_LIMIT, structlog).await { + Err(ellapsed_error) => Err(RpcError::Transport(TransportErrorKind::Custom(Box::new( + ellapsed_error, + )))), + Ok(structlog_res) => Ok(normalize_structlog(structlog_res?).await), + } } /// Provides a way to check in constant time if an address points to a @@ -119,7 +134,7 @@ pub(crate) fn generate_jumpdest_table( ); callee_addr_to_code_hash[&to_address] } else { - let init = tx.input.clone(); + let init = &tx.input; keccak(init) }; @@ -240,33 +255,36 @@ pub(crate) fn generate_jumpdest_table( ensure!(*offset <= U256::from(usize::MAX)); let offset: usize = offset.to(); ensure!(*size <= U256::from(usize::MAX)); + let size: usize = size.to(); - let memory_size = entry.memory.as_ref().unwrap().len(); - let salt: [u8; 32] = salt.to_be_bytes(); + let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; + let salt: Word = salt.to_be_bytes(); ensure!( - entry.memory.is_some() && size <= memory_size, - "No or insufficient memory available for {op}. Contract size is {size} while memory size is {memory_size}." + entry.memory.is_some() && offset + size <= memory_size, + "Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size ); let memory_raw: &[String] = entry.memory.as_ref().unwrap(); - let memory_parsed: Vec> = memory_raw + let memory_parsed: Vec> = memory_raw .iter() .map(|s| { - let c = s.parse(); + // let c = s.parse(); + let c = U256::from_str_radix(s, 16); ensure!(c.is_ok(), "Parsing memory failed."); let a: U256 = c.unwrap(); - let d: [u8; 32] = a.to_be_bytes(); + let d: Word = a.to_be_bytes(); Ok(d) }) .collect(); - let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); + let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); let memory: Vec = mem_res?.concat(); let init_code = &memory[offset..offset + size]; - let contract_address = tx.from.create2_from_code(salt, init_code); - ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); - let code_hash = callee_addr_to_code_hash[&contract_address]; - call_stack.push((code_hash, next_ctx_available)); + let init_code_hash = keccak(init_code); + // let contract_address = tx.from.create2_from_code(salt, init_code); + // ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); + // let code_hash = callee_addr_to_code_hash[&contract_address]; + call_stack.push((init_code_hash, next_ctx_available)); next_ctx_available += 1; prev_jump = None; @@ -286,7 +304,9 @@ pub(crate) fn generate_jumpdest_table( }; ensure!( *counter <= U256::from(u64::MAX), - "Operand for {op} caused overflow." + "Operand for {op} caused overflow: counter: {} is larger than u64::MAX {}", + *counter, + u64::MAX ); let jump_target: u64 = counter.to(); diff --git a/zero_bin/rpc/src/native/txn.rs b/zero_bin/rpc/src/native/txn.rs index 80d2f367b..63e61d7e2 100644 --- a/zero_bin/rpc/src/native/txn.rs +++ b/zero_bin/rpc/src/native/txn.rs @@ -137,7 +137,7 @@ where tx_receipt_fut, pre_trace_fut, diff_trace_fut, - structlog_trace_fut + structlog_trace_fut, )?; Ok(( diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index f3d128d46..e00e79e8d 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -35,7 +35,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_MIN_STACK=33554432 #export RUST_BACKTRACE=full -#export RUST_LOG=info +#export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' @@ -102,7 +102,7 @@ if [[ $TEST_ONLY == "test_only" ]]; then rm $TEST_OUT_PATH exit else - echo "Failed to create proof witnesses. See \"zk_evm/tools/test.out\" for more details." + echo "Failed to create proof witnesses. See \"zk_evm/zero_bin/tools/test.out\" for more details." exit 1 fi fi From 8a964b8bd2442c27b1f59c3dd61e3e502f5a80d2 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 18 Sep 2024 16:42:04 +0200 Subject: [PATCH 014/112] fmt, clippy --- .../src/generation/prover_input.rs | 14 ++++++++------ test_jerigon.sh | 2 +- zero_bin/rpc/src/jerigon.rs | 15 +++++++++++---- zero_bin/rpc/src/jumpdest.rs | 4 ++-- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 1919f98d2..77aaec99d 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -775,18 +775,20 @@ impl GenerationState { let rpcw = &self.inputs.jumpdest_tables; let rpc: Option = rpcw .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(&jdt, &self.inputs.contract_code)); + .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); let (sim, simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) && rw != &sw { - info!("SIMW {}", sw); - info!("RPCW {}", rw); - assert_eq!(rw, &sw); - } + if let (Some(rw), Some(sw)) = (rpcw, simw) + && rw != &sw + { + info!("SIMW {}", sw); + info!("RPCW {}", rw); + assert_eq!(rw, &sw); + } self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; diff --git a/test_jerigon.sh b/test_jerigon.sh index 5362d6156..fd7800872 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -152,7 +152,7 @@ RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` #BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" -BLOCKS="$DECODING" +BLOCKS="$DECODING $CREATE2" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 0c3900ab8..82feee7a0 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -169,12 +169,19 @@ where let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { - debug!("{:#?}: JumpDestTable generation failed with reason: {}", tx.hash, error); + debug!( + "{:#?}: JumpDestTable generation failed with reason: {}", + tx.hash, error + ); None }, - |jdt|{ - debug!("{:#?}: JumpDestTable generation succeceeded with result: {}",tx.hash, jdt); - Some(jdt)}, + |jdt| { + debug!( + "{:#?}: JumpDestTable generation succeceeded with result: {}", + tx.hash, jdt + ); + Some(jdt) + }, ) }); diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 321acd9fa..8e3ffe8f4 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -249,7 +249,7 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [_value, offset, size, salt, ..] = evm_stack[..] else { + let [_value, offset, size, _salt, ..] = evm_stack[..] else { unreachable!() }; ensure!(*offset <= U256::from(usize::MAX)); @@ -258,7 +258,7 @@ pub(crate) fn generate_jumpdest_table( let size: usize = size.to(); let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; - let salt: Word = salt.to_be_bytes(); + // let salt: Word = salt.to_be_bytes(); ensure!( entry.memory.is_some() && offset + size <= memory_size, From 32e68bfa33c8ae3c0361ba85c9e61039225a32ae Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 19 Sep 2024 13:46:54 +0200 Subject: [PATCH 015/112] investigate 15,35 --- .../src/generation/prover_input.rs | 5 ++- test_jerigon.sh | 26 ++++++++++-- trace_decoder/src/core.rs | 1 + zero_bin/rpc/src/jerigon.rs | 2 +- zero_bin/rpc/src/jumpdest.rs | 40 ++++++++++--------- zero_bin/tools/prove_stdio.sh | 2 +- 6 files changed, 50 insertions(+), 26 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 77aaec99d..c2b1d2169 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -790,7 +790,10 @@ impl GenerationState { assert_eq!(rw, &sw); } - self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + info!("SIMW {:#?}", sim); + info!("RPCW {:#?}", rpc); + + self.jumpdest_table = sim;// = if rpc.is_some() { rpc } else { sim }; Ok(()) } diff --git a/test_jerigon.sh b/test_jerigon.sh index fd7800872..a1a1c85ca 100755 --- a/test_jerigon.sh +++ b/test_jerigon.sh @@ -11,9 +11,9 @@ fi mkdir -p witnesses export RAYON_NUM_THREADS=4 -export TOKIO_WORKER_THREADS=4 +export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUST_LOG=info +export RUST_LOG=rpc=trace,evm_arithmetization::generation::prover_input=trace export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' export RUST_MIN_STACK=67108864 @@ -137,6 +137,23 @@ USEDTOFAIL=" 77 " +ROUND2=" +15 +35 +566 +664 +665 +667 +670 +" +#444 + +TESTED=" +4 +5 +28 +65 +" # 470..663 from Robin for i in {470..663} @@ -152,7 +169,8 @@ RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" #BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` #BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" -BLOCKS="$DECODING $CREATE2" +#BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI" +BLOCKS="$ROUND2" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` #echo "Testing: $BLOCKS" @@ -164,7 +182,7 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace - timeout 2m cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 2cfb934ed..d227edfb8 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -76,6 +76,7 @@ pub fn entrypoint( withdrawals, )?; + dbg!(&batches.first().unwrap().jumpdest_tables); let mut running_gas_used = 0; Ok(batches .into_iter() diff --git a/zero_bin/rpc/src/jerigon.rs b/zero_bin/rpc/src/jerigon.rs index 82feee7a0..b3742f27f 100644 --- a/zero_bin/rpc/src/jerigon.rs +++ b/zero_bin/rpc/src/jerigon.rs @@ -177,7 +177,7 @@ where }, |jdt| { debug!( - "{:#?}: JumpDestTable generation succeceeded with result: {}", + "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); Some(jdt) diff --git a/zero_bin/rpc/src/jumpdest.rs b/zero_bin/rpc/src/jumpdest.rs index 8e3ffe8f4..4861aeeca 100644 --- a/zero_bin/rpc/src/jumpdest.rs +++ b/zero_bin/rpc/src/jumpdest.rs @@ -1,4 +1,5 @@ use core::default::Default; +use core::option::Option::None; use core::time::Duration; use std::collections::BTreeMap; use std::collections::HashMap; @@ -97,7 +98,7 @@ fn precompiles() -> &'static HashSet
{ /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. -#[instrument] +// #[instrument] pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], @@ -107,6 +108,7 @@ pub(crate) fn generate_jumpdest_table( let mut jumpdest_table = JumpDestTableWitness::default(); + // This does not contain `initcodes`. let callee_addr_to_code_hash: HashMap = tx_traces .iter() .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) @@ -126,16 +128,16 @@ pub(crate) fn generate_jumpdest_table( } ); - let entrypoint_code_hash: H256 = if let Some(to_address) = tx.to { - // Guard against transactions to a non-contract address. - ensure!( - callee_addr_to_code_hash.contains_key(&to_address), - format!("Callee addr {} is not at contract address", to_address) - ); - callee_addr_to_code_hash[&to_address] - } else { - let init = &tx.input; - keccak(init) + let entrypoint_code_hash: H256 = match tx.to { + Some(to_address) if precompiles().contains(&to_address) => return Ok(jumpdest_table), + Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { + return Ok(jumpdest_table) + } + Some(to_address) => callee_addr_to_code_hash[&to_address], + None => { + let init = &tx.input; + keccak(init) + } }; // `None` encodes that previous `entry` was not a JUMP or JUMPI with true @@ -165,14 +167,14 @@ pub(crate) fn generate_jumpdest_table( ensure!(call_stack.is_empty().not(), "Call stack was empty."); let (code_hash, ctx) = call_stack.last().unwrap(); - trace!("TX: {:?}", tx.hash); - trace!("STEP: {:?}", step); - trace!("STEPS: {:?}", struct_log.len()); - trace!("OPCODE: {}", entry.op.as_str()); - trace!("CODE: {:?}", code_hash); - trace!("CTX: {:?}", ctx); - trace!("CURR_DEPTH: {:?}", curr_depth); - trace!("{:#?}\n", entry); + // trace!("TX: {:?}", tx.hash); + // trace!("STEP: {:?}", step); + // trace!("STEPS: {:?}", struct_log.len()); + // trace!("OPCODE: {}", entry.op.as_str()); + // trace!("CODE: {:?}", code_hash); + // trace!("CTX: {:?}", ctx); + // trace!("CURR_DEPTH: {:?}", curr_depth); + // trace!("{:#?}\n", entry); match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { diff --git a/zero_bin/tools/prove_stdio.sh b/zero_bin/tools/prove_stdio.sh index e00e79e8d..88bc359fc 100755 --- a/zero_bin/tools/prove_stdio.sh +++ b/zero_bin/tools/prove_stdio.sh @@ -96,7 +96,7 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH + cargo run --quiet --release --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --batch-size $BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" rm $TEST_OUT_PATH From 184878dda760dd48cbf400d23407e69cacd4fbe0 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 19 Sep 2024 21:08:52 +0200 Subject: [PATCH 016/112] fix scripts --- scripts/prove_stdio.sh | 2 +- scripts/test_jerigon.sh | 6 +++--- scripts/test_native.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index cffb39a34..80515c9e1 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -40,7 +40,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index d81fd3067..240148594 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -138,8 +138,6 @@ USEDTOFAIL=" " ROUND2=" -15 -35 664 665 667 @@ -153,6 +151,8 @@ NOWSUCCESS=" 28 65 566 +15 +35 " # 470..663 from Robin @@ -186,7 +186,7 @@ for BLOCK in $BLOCKS; do echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - scripts/prove_stdio.sh $WITNESS test_only + prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 088205b22..d2a7aac5f 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -69,7 +69,7 @@ echo "Testing prepared witnesses.." for WITNESS in witnesses/*.native.$GITHASH.witness.json; do echo "Testing $WITNESS" - zero_bin/tools/prove_stdio.sh $WITNESS test_only + prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then From c000b5a55d7bccdeed29da60559533114279aa77 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 19 Sep 2024 22:46:57 +0200 Subject: [PATCH 017/112] remove redtape for JUMP/I --- .../src/cpu/kernel/tests/mod.rs | 2 +- .../src/generation/prover_input.rs | 6 +- scripts/prove_stdio.sh | 2 +- scripts/test_jerigon.sh | 5 +- zero/src/rpc/jerigon.rs | 11 ++-- zero/src/rpc/jumpdest.rs | 58 ++++++++++--------- zero/src/rpc/native/txn.rs | 2 +- 7 files changed, 45 insertions(+), 41 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/tests/mod.rs b/evm_arithmetization/src/cpu/kernel/tests/mod.rs index aae20ae78..f4fb947e6 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/mod.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/mod.rs @@ -26,9 +26,9 @@ mod signed_syscalls; mod transaction_parsing; mod transient_storage; -use ::core::iter::Iterator as _; use std::{ops::Range, str::FromStr}; +use ::core::iter::Iterator as _; use anyhow::Result; use ethereum_types::U256; use plonky2::hash::hash_types::RichField; diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index e2ea4b179..c561504df 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -790,10 +790,10 @@ impl GenerationState { assert_eq!(rw, &sw); } - info!("SIMW {:#?}", sim); - info!("RPCW {:#?}", rpc); + info!("SIM {:#?}", sim); + info!("RPC {:#?}", rpc); - self.jumpdest_table = sim;// = if rpc.is_some() { rpc } else { sim }; + self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; Ok(()) } diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 80515c9e1..dc59806b9 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -40,7 +40,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 240148594..baee31ea7 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -13,8 +13,7 @@ mkdir -p witnesses export RAYON_NUM_THREADS=4 export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUST_LOG=rpc=trace,evm_arithmetization::generation::prover_input=trace -export 'RUSTFLAGS=-C target-cpu=native -Zlinker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` @@ -142,10 +141,10 @@ ROUND2=" 665 667 670 -444 " NOWSUCCESS=" +444 4 5 28 diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 9a9946a23..ab8cf8d37 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -16,9 +16,12 @@ use serde_json::json; use trace_decoder::{ BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, }; -use tracing::debug; +use tracing::info; -use super::{fetch_other_block_data, jumpdest::{self, get_normalized_structlog}}; +use super::{ + fetch_other_block_data, + jumpdest::{self, get_normalized_structlog}, +}; use crate::prover::BlockProverInput; use crate::provider::CachedProvider; /// Transaction traces retrieved from Erigon zeroTracer. @@ -167,14 +170,14 @@ where let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { - debug!( + info!( "{:#?}: JumpDestTable generation failed with reason: {}", tx.hash, error ); None }, |jdt| { - debug!( + info!( "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 4861aeeca..533a9229b 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -143,7 +143,7 @@ pub(crate) fn generate_jumpdest_table( // `None` encodes that previous `entry` was not a JUMP or JUMPI with true // condition, `Some(jump_target)` encodes we came from a JUMP or JUMPI with // true condition and target `jump_target`. - let mut prev_jump = None; + let mut prev_jump: Option = None; // Call depth of the previous `entry`. We initialize to 0 as this compares // smaller to 1. @@ -167,14 +167,14 @@ pub(crate) fn generate_jumpdest_table( ensure!(call_stack.is_empty().not(), "Call stack was empty."); let (code_hash, ctx) = call_stack.last().unwrap(); - // trace!("TX: {:?}", tx.hash); - // trace!("STEP: {:?}", step); - // trace!("STEPS: {:?}", struct_log.len()); - // trace!("OPCODE: {}", entry.op.as_str()); - // trace!("CODE: {:?}", code_hash); - // trace!("CTX: {:?}", ctx); - // trace!("CURR_DEPTH: {:?}", curr_depth); - // trace!("{:#?}\n", entry); + trace!("TX: {:?}", tx.hash); + trace!("STEP: {:?}", step); + trace!("STEPS: {:?}", struct_log.len()); + trace!("OPCODE: {}", entry.op.as_str()); + trace!("CODE: {:?}", code_hash); + trace!("CTX: {:?}", ctx); + trace!("CURR_DEPTH: {:?}", curr_depth); + trace!("{:#?}\n", entry); match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { @@ -226,7 +226,7 @@ pub(crate) fn generate_jumpdest_table( let operands = 3; ensure!( evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", + "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); let [_value, _offset, _size, ..] = evm_stack[..] else { @@ -301,18 +301,18 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [counter, ..] = evm_stack[..] else { + let [jump_target, ..] = evm_stack[..] else { unreachable!() }; - ensure!( - *counter <= U256::from(u64::MAX), - "Operand for {op} caused overflow: counter: {} is larger than u64::MAX {}", - *counter, - u64::MAX - ); - let jump_target: u64 = counter.to(); - - prev_jump = Some(jump_target); + // ensure!( + // *counter <= U256::from(u64::MAX), + // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX + // {}", *counter, + // u64::MAX + // ); + // let jump_target: u64 = counter.to(); + + prev_jump = Some(*jump_target); } "JUMPI" => { ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); @@ -324,25 +324,27 @@ pub(crate) fn generate_jumpdest_table( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - let [counter, condition, ..] = evm_stack[..] else { + let [jump_target, condition, ..] = evm_stack[..] else { unreachable!() }; - ensure!( - *counter <= U256::from(u64::MAX), - "Operand for {op} caused overflow." - ); - let jump_target: u64 = counter.to(); + // ensure!( + // *counter <= U256::from(u64::MAX), + // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX + // {}", *counter, + // u64::MAX + // ); + // let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); prev_jump = if jump_condition { - Some(jump_target) + Some(*jump_target) } else { None }; } "JUMPDEST" => { let jumped_here = if let Some(jmp_target) = prev_jump { - jmp_target == entry.pc + jmp_target == U256::from(entry.pc) } else { false }; diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index fdfdf23f5..1305f9010 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -1,6 +1,5 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use crate::rpc::Compat; use __compat_primitive_types::{H256, U256}; use alloy::{ primitives::{keccak256, Address, B256}, @@ -26,6 +25,7 @@ use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use tracing::debug; use crate::rpc::jumpdest::{self, get_normalized_structlog}; +use crate::rpc::Compat; /// Processes the transactions in the given block and updates the code db. pub async fn process_transactions( From ec817013e670ae78b7fedc95dd28f27de5ca7e81 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 00:12:35 +0200 Subject: [PATCH 018/112] misc --- scripts/prove_stdio.sh | 2 +- scripts/test_jerigon.sh | 8 ++++---- zero/src/rpc/jumpdest.rs | 3 +-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index dc59806b9..f525d8b4c 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -40,7 +40,7 @@ export RAYON_NUM_THREADS=$num_procs #export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' +#export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index baee31ea7..fe2eba2fa 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -13,8 +13,8 @@ mkdir -p witnesses export RAYON_NUM_THREADS=4 export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' -export RUST_MIN_STACK=67108864 +#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +#export RUST_MIN_STACK=67108864 GITHASH=`git rev-parse --short HEAD` echo "Testing against jergion, current revision: $GITHASH." @@ -138,9 +138,9 @@ USEDTOFAIL=" ROUND2=" 664 -665 667 670 +665 " NOWSUCCESS=" @@ -185,7 +185,7 @@ for BLOCK in $BLOCKS; do echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - prove_stdio.sh $WITNESS test_only + ./prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 533a9229b..d6dc4579d 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -26,7 +26,7 @@ use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::TxnTrace; -use tracing::{instrument, trace}; +use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. @@ -98,7 +98,6 @@ fn precompiles() -> &'static HashSet
{ /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. -// #[instrument] pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], From bff471ed894b97df98171abdafbf2f33b8ee835a Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 11:49:25 +0200 Subject: [PATCH 019/112] fix ci --- scripts/prove_stdio.sh | 11 +++++------ scripts/test_jerigon.sh | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index f525d8b4c..8c5e60fe9 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -30,17 +30,16 @@ TEST_OUT_PATH="${REPO_ROOT}/test.out" BLOCK_BATCH_SIZE=${BLOCK_BATCH_SIZE:=1} - # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs -#export TOKIO_WORKER_THREADS=$num_procs +export TOKIO_WORKER_THREADS=$num_procs -#export RUST_MIN_STACK=33554432 -#export RUST_BACKTRACE=full -#export RUST_LOG=trace +export RUST_MIN_STACK=33554432 +export RUST_BACKTRACE=full +export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. -#export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' +export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' INPUT_FILE=$1 TEST_ONLY=$2 diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index fe2eba2fa..5bb8e7293 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -2,7 +2,6 @@ set -uo pipefail -#export CARGO_LOG=cargo::core::compiler::fingerprint=debug export RPC= if [ -z $RPC ]; then # You must set an RPC endpoint From ca9620dc26c01de7a77540323317b30e36fa180f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 16:35:39 +0200 Subject: [PATCH 020/112] minimize diff --- .gitignore | 1 - .../benches/fibonacci_25m_gas.rs | 2 +- .../src/cpu/kernel/interpreter.rs | 2 +- .../src/cpu/kernel/tests/add11.rs | 4 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 2 +- .../src/cpu/kernel/tests/mod.rs | 1 - .../src/generation/jumpdest.rs | 16 +------ evm_arithmetization/src/generation/mod.rs | 6 +-- .../src/generation/prover_input.rs | 16 +++---- evm_arithmetization/tests/add11_yml.rs | 2 +- evm_arithmetization/tests/erc20.rs | 2 +- evm_arithmetization/tests/erc721.rs | 2 +- evm_arithmetization/tests/log_opcode.rs | 2 +- evm_arithmetization/tests/selfdestruct.rs | 2 +- evm_arithmetization/tests/simple_transfer.rs | 2 +- evm_arithmetization/tests/withdrawals.rs | 2 +- scripts/prove_stdio.sh | 9 ++-- scripts/test_jerigon.sh | 11 +---- trace_decoder/src/core.rs | 18 ++++---- trace_decoder/src/interface.rs | 2 +- trace_decoder/src/jumpdest.rs | 1 - trace_decoder/src/lib.rs | 1 - zero/src/bin/rpc.rs | 2 +- zero/src/rpc/jumpdest.rs | 43 +++++++------------ zero/src/rpc/native/mod.rs | 3 +- zero/src/rpc/native/txn.rs | 15 +++++-- 26 files changed, 71 insertions(+), 98 deletions(-) delete mode 100644 trace_decoder/src/jumpdest.rs diff --git a/.gitignore b/.gitignore index d262be31e..9f5ee7a64 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ # Misc ###### /**/*.ignoreme -**/output.log /**/*.ipynb /**/*.log /**/*.out diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index 2b35b2a1c..106608935 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -193,7 +193,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 85d43771f..9c2f7bfac 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -118,9 +118,9 @@ pub(crate) struct ExtraSegmentData { pub(crate) withdrawal_prover_inputs: Vec, pub(crate) ger_prover_inputs: Vec, pub(crate) trie_root_ptrs: TrieRootPtrs, + pub(crate) jumpdest_table: Option, pub(crate) accounts: BTreeMap, pub(crate) storage: BTreeMap<(U256, U256), usize>, - pub(crate) jumpdest_table: Option, pub(crate) next_txn_index: usize, } diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index fbae6647d..4a6dc9447 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -194,7 +194,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let initial_stack = vec![]; @@ -372,7 +372,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index 673b516b0..716e238d0 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,7 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/mod.rs b/evm_arithmetization/src/cpu/kernel/tests/mod.rs index f4fb947e6..2b983d099 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/mod.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/mod.rs @@ -28,7 +28,6 @@ mod transient_storage; use std::{ops::Range, str::FromStr}; -use ::core::iter::Iterator as _; use anyhow::Result; use ethereum_types::U256; use plonky2::hash::hash_types::RichField; diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index fc2a8e1a6..9a3201cce 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -51,18 +51,11 @@ impl JumpDestTableWitness { /// Creates the required `ctx` keys and `code_hash`. Idempotent. pub fn insert(&mut self, code_hash: H256, ctx: usize, offset: usize) { (*self).entry(code_hash).or_default().insert(ctx, offset); - - // TODO(einar) remove before publishing PR. - assert!(self.0.contains_key(&code_hash)); - assert!(self.0[&code_hash].0.contains_key(&ctx)); - assert!(self.0[&code_hash].0[&ctx].contains(&offset)); } pub fn extend(mut self, other: &Self, prev_max_ctx: usize) -> (Self, usize) { let mut curr_max_ctx = prev_max_ctx; - // TODO: Opportunity for optimization: Simulate to generate only missing - // JUMPDEST tables. for (code_hash, ctx_tbl) in (*other).iter() { for (ctx, jumpdests) in ctx_tbl.0.iter() { let batch_ctx = prev_max_ctx + ctx; @@ -70,12 +63,7 @@ impl JumpDestTableWitness { for offset in jumpdests { self.insert(*code_hash, batch_ctx, *offset); - - assert!(self.0.contains_key(code_hash)); - assert!(self.0[code_hash].0.contains_key(&batch_ctx)); - assert!(self.0[code_hash].0[&batch_ctx].contains(offset)); } - // dbg!(&self); } } @@ -90,7 +78,7 @@ impl JumpDestTableWitness { impl Display for JumpDestTableWitness { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "=== JumpDestTableWitness ===")?; + writeln!(f, "\n=== JumpDestTableWitness ===")?; for (code, ctxtbls) in &self.0 { write!(f, "codehash: {:#x}\n{}", code, ctxtbls)?; @@ -115,7 +103,7 @@ impl Display for ContextJumpDests { impl Display for JumpDestTableProcessed { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "=== JumpDestTableProcessed ===")?; + writeln!(f, "\n=== JumpDestTableProcessed ===")?; let v = sorted(self.0.clone()); for (ctx, code) in v { diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index e5ec7bde3..0a775eed5 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -111,7 +111,7 @@ pub struct GenerationInputs { /// A table listing each JUMPDESTs reached in each call context under /// associated code hash. - pub jumpdest_tables: Option, + pub jumpdest_table: Option, } /// A lighter version of [`GenerationInputs`], which have been trimmed @@ -165,7 +165,7 @@ pub struct TrimmedGenerationInputs { /// A list of tables listing each JUMPDESTs reached in each call context /// under associated code hash. - pub jumpdest_tables: Option, + pub jumpdest_table: Option, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] @@ -240,7 +240,7 @@ impl GenerationInputs { burn_addr: self.burn_addr, block_metadata: self.block_metadata.clone(), block_hashes: self.block_hashes.clone(), - jumpdest_tables: self.jumpdest_tables.clone(), + jumpdest_table: self.jumpdest_table.clone(), } } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index c561504df..e30ea3b6e 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -43,8 +43,7 @@ use crate::witness::memory::MemoryAddress; use crate::witness::operation::CONTEXT_SCALING_FACTOR; use crate::witness::util::{current_context_peek, stack_peek}; -/// A set of contract code as a byte arrays. From this a mapping: hash -> -/// contract can be built. +/// A set to hold contract code as a byte vectors. pub type CodeDb = BTreeSet>; /// Prover input function represented as a scoped function name. @@ -768,11 +767,11 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - dbg!(&self.inputs.jumpdest_tables); + dbg!(&self.inputs.jumpdest_table); // w for witness // let txn_idx = self.next_txn_index - 1; // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code - let rpcw = &self.inputs.jumpdest_tables; + let rpcw = &self.inputs.jumpdest_table; let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); @@ -809,13 +808,10 @@ impl GenerationState { self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); - trace!( - "ctx: {ctx}, code_hash: {:?} code: {:?}", - keccak(code.clone()), - code - ); + let code_hash = keccak(code.clone()); + trace!("ctx: {ctx}, code_hash: {:?} code: {:?}", code_hash, code); for offset in jumpdest_table.clone() { - jdtw.insert(keccak(code.clone()), ctx, offset); + jdtw.insert(code_hash, ctx, offset); } if let Some(&largest_address) = jumpdest_table.last() { let proofs = get_proofs_and_jumpdests(&code, largest_address, jumpdest_table); diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 12a84fbe7..f3af1539e 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,7 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index 475d03a8e..d251ed6fc 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -196,7 +196,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index 8d30f85d1..576eb1e57 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -200,7 +200,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index 951ebbb28..9de1e5d63 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -267,7 +267,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 221113b26..f3e6996c1 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -171,7 +171,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index de8e096cf..933f3fd2c 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -163,7 +163,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index 2aa776e54..6f4686ef6 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -106,7 +106,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_tables: Default::default(), + jumpdest_table: Default::default(), }; let max_cpu_len_log = 20; diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 8c5e60fe9..2b804f98a 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -21,6 +21,9 @@ fi REPO_ROOT=$(git rev-parse --show-toplevel) PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs" +BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}" +echo "Block batch size: $BLOCK_BATCH_SIZE" + BATCH_SIZE=${BATCH_SIZE:=1} echo "Batch size: $BATCH_SIZE" @@ -28,15 +31,13 @@ OUTPUT_LOG="${REPO_ROOT}/output.log" PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" TEST_OUT_PATH="${REPO_ROOT}/test.out" -BLOCK_BATCH_SIZE=${BLOCK_BATCH_SIZE:=1} - # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs export TOKIO_WORKER_THREADS=$num_procs export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full -export RUST_LOG=trace +export RUST_LOG=info # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' @@ -103,7 +104,7 @@ if [[ $TEST_ONLY == "test_only" ]]; then rm $TEST_OUT_PATH exit else - echo "Failed to create proof witnesses. See \"zk_evm/zero_bin/tools/test.out\" for more details." + echo "Failed to create proof witnesses. See \"zk_evm/test.out\" for more details." exit 1 fi fi diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 5bb8e7293..6d1125216 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -163,15 +163,10 @@ TIP=688 NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` -# CIBLOCKS="$KNOWNFAILED $ROBIN $RANDOMBLOCKS $CIBLOCKS" -#BLOCKS="$ROBIN $RANDOMBLOCKS $CIBLOCKS" -#BLOCKS=`echo $CIBLOCKS | sed 's/\s/\n/g'` -#BLOCKS="$CIBLOCKS $KNOWNFAILED $RANDOMBLOCKS" -#BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI" -BLOCKS="$ROUND2" +BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` -#echo "Testing: $BLOCKS" +echo "Testing: $BLOCKS" printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt echo "---------------------------" | tee -a witnesses/jerigon_results.txt @@ -194,5 +189,3 @@ for BLOCK in $BLOCKS; do fi printf "%s %10i %s\n" $GITHASH $BLOCK $RESULT | tee -a witnesses/jerigon_results.txt done - -exit 0 diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 2b1a0e49c..cd410e58b 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -77,7 +77,6 @@ pub fn entrypoint( withdrawals, )?; - dbg!(&batches.first().unwrap().jumpdest_tables); let mut running_gas_used = 0; Ok(batches .into_iter() @@ -117,32 +116,35 @@ pub fn entrypoint( checkpoint_state_trie_root, checkpoint_consolidated_hash, contract_code: { - let cc = contract_code.into_iter(); - let initcodes = byte_code .iter() - .filter_map(|nonempty_txn_raw| -> Option> { + .filter_map(|nonempty_txn_bytes| -> Option> { let tx_envelope = - TxEnvelope::decode(&mut &nonempty_txn_raw[..]).unwrap(); + TxEnvelope::decode(&mut &nonempty_txn_bytes[..]).unwrap(); match tx_envelope.to() { TxKind::Create => Some(tx_envelope.input().to_vec()), TxKind::Call(_address) => None, } }); - cc.chain(initcodes) + contract_code + .into_iter() + .chain(initcodes) .map(|it| (keccak_hash::keccak(&it), it)) .collect() }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), burn_addr, - jumpdest_tables: { + jumpdest_table: { + // Note that this causes any batch containing just a single `None` to collapse + // into a `None`, which causing failover to simulating jumpdest analysis for the + // whole batch. There is an optimization opportunity here. jumpdest_tables .into_iter() .collect::>>() - .map(|vj| JumpDestTableWitness::merge(vj.iter()).0) + .map(|jdt| JumpDestTableWitness::merge(jdt.iter()).0) }, }, ) diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index 5c72d6b3f..81cc87bbe 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -170,7 +170,7 @@ pub enum ContractCodeUsage { /// contract code will not appear in the [`BlockTrace`] map. Write(#[serde(with = "crate::hex")] Vec), } -// Question: Why has this has been removed upstream. Proably unused. +// Review: Re-adding this. It has been removed upstream. impl ContractCodeUsage { /// Get code hash from a read or write operation of contract code. pub fn get_code_hash(&self) -> H256 { diff --git a/trace_decoder/src/jumpdest.rs b/trace_decoder/src/jumpdest.rs deleted file mode 100644 index 8b1378917..000000000 --- a/trace_decoder/src/jumpdest.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index aaab848d4..8c54eea8e 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -60,7 +60,6 @@ use plonky2::field::goldilocks_field::GoldilocksField; mod type1; // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 // add backend/prod support for type 2 -mod jumpdest; #[cfg(test)] #[allow(dead_code)] mod type2; diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index d9f25901a..b9751196c 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -207,7 +207,7 @@ async fn main() -> anyhow::Result<()> { .with( tracing_subscriber::fmt::layer() // With the default configuration trace information is written - // to stdout, but we already use to write our payload (the witness). + // to stdout, but we already use stdout to write our payload (the witness). .with_writer(std::io::stderr) .with_ansi(false) .compact() diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index d6dc4579d..a48d4759a 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -65,11 +65,12 @@ where TransportT: Transport + Clone, { // Optimization: It may be a better default to pull the stack immediately. - let light_structlog_trace = provider - .debug_trace_transaction(*tx_hash, structlog_tracing_options(false, false, false)) + let stackonly_structlog_trace = provider + .debug_trace_transaction(*tx_hash, structlog_tracing_options(true, false, false)) .await?; - let structlogs_opt: Option> = normalize_structlog(light_structlog_trace).await; + let structlogs_opt: Option> = + normalize_structlog(stackonly_structlog_trace).await; let need_memory = structlogs_opt.is_some_and(trace_contains_create2); trace!("Need structlog with memory: {need_memory}"); @@ -107,7 +108,7 @@ pub(crate) fn generate_jumpdest_table( let mut jumpdest_table = JumpDestTableWitness::default(); - // This does not contain `initcodes`. + // This map does not contain `initcodes`. let callee_addr_to_code_hash: HashMap = tx_traces .iter() .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) @@ -192,22 +193,24 @@ pub(crate) fn generate_jumpdest_table( }; let callee_address = { - // Clear the upper half of the operand. let callee_raw = *address; - // let (callee_raw, _overflow) = callee_raw.overflowing_shl(128); - // let (callee_raw, _overflow) = callee_raw.overflowing_shr(128); - ensure!(callee_raw <= U256::from(U160::MAX)); let lower_20_bytes = U160::from(callee_raw); Address::from(lower_20_bytes) }; + if callee_addr_to_code_hash.contains_key(&callee_address) { + let next_code_hash = callee_addr_to_code_hash[&callee_address]; + call_stack.push((next_code_hash, next_ctx_available)); + }; + if precompiles().contains(&callee_address) { trace!("Called precompile at address {}.", &callee_address); - } else if callee_addr_to_code_hash.contains_key(&callee_address) { - let code_hash = callee_addr_to_code_hash[&callee_address]; - call_stack.push((code_hash, next_ctx_available)); - } else { + }; + + if callee_addr_to_code_hash.contains_key(&callee_address).not() + && precompiles().contains(&callee_address).not() + { // This case happens if calling an EOA. This is described // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun trace!( @@ -303,13 +306,6 @@ pub(crate) fn generate_jumpdest_table( let [jump_target, ..] = evm_stack[..] else { unreachable!() }; - // ensure!( - // *counter <= U256::from(u64::MAX), - // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX - // {}", *counter, - // u64::MAX - // ); - // let jump_target: u64 = counter.to(); prev_jump = Some(*jump_target); } @@ -326,13 +322,6 @@ pub(crate) fn generate_jumpdest_table( let [jump_target, condition, ..] = evm_stack[..] else { unreachable!() }; - // ensure!( - // *counter <= U256::from(u64::MAX), - // "Operand for {op} caused overflow: counter: {} is larger than u64::MAX - // {}", *counter, - // u64::MAX - // ); - // let jump_target: u64 = counter.to(); let jump_condition = condition.is_zero().not(); prev_jump = if jump_condition { @@ -347,7 +336,7 @@ pub(crate) fn generate_jumpdest_table( } else { false }; - let jumpdest_offset = entry.pc as usize; + let jumpdest_offset = TryInto::::try_into(entry.pc)?; if jumped_here { jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset); } diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 11766b148..5563cad5b 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -15,8 +15,7 @@ use crate::provider::CachedProvider; mod state; mod txn; -pub use txn::process_transaction; -pub use txn::process_transactions; +pub use txn::{process_transaction, process_transactions}; /// Fetches the prover input for the given BlockId. pub async fn block_prover_input( diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 1305f9010..34ed820df 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -22,7 +22,7 @@ use anyhow::{Context as _, Ok}; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; -use tracing::debug; +use tracing::info; use crate::rpc::jumpdest::{self, get_normalized_structlog}; use crate::rpc::Compat; @@ -86,10 +86,19 @@ where let jumpdest_table: Option = structlog_opt.and_then(|struct_logs| { jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces).map_or_else( |error| { - debug!("JumpDestTable generation failed with reason: {}", error); + info!( + "{:#?}: JumpDestTable generation failed with reason: {}", + tx.hash, error + ); None }, - Some, + |jdt| { + info!( + "{:#?}: JumpDestTable generation succeeded with result: {}", + tx.hash, jdt + ); + Some(jdt) + }, ) }); From 4c97c0f8537aca0b279933286b13100890ac8a57 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 17:14:19 +0200 Subject: [PATCH 021/112] include whole function in timeout --- zero/src/rpc/jumpdest.rs | 48 +++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index a48d4759a..f44aa7a43 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -51,7 +51,7 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } -fn trace_contains_create2(structlog: Vec) -> bool { +fn trace_contains_create2(structlog: &[StructLog]) -> bool { structlog.iter().any(|entry| entry.op == "CREATE2") } @@ -64,27 +64,39 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - // Optimization: It may be a better default to pull the stack immediately. - let stackonly_structlog_trace = provider - .debug_trace_transaction(*tx_hash, structlog_tracing_options(true, false, false)) - .await?; + let inner = async { + // Optimization: It may be a better default to pull the stack immediately. + let stackonly_structlog_trace = provider + .debug_trace_transaction(*tx_hash, structlog_tracing_options(true, false, false)) + .await?; - let structlogs_opt: Option> = - normalize_structlog(stackonly_structlog_trace).await; + let stackonly_structlog_opt: Option> = + normalize_structlog(&stackonly_structlog_trace).await; - let need_memory = structlogs_opt.is_some_and(trace_contains_create2); - trace!("Need structlog with memory: {need_memory}"); + let need_memory = stackonly_structlog_opt + .as_deref() + .is_some_and(trace_contains_create2); + trace!("Need structlog with memory: {need_memory}"); - let structlog = provider.debug_trace_transaction( - *tx_hash, - structlog_tracing_options(true, need_memory, false), - ); + if need_memory.not() { + return Ok(stackonly_structlog_opt); + }; + + let memory_structlog_fut = provider.debug_trace_transaction( + *tx_hash, + structlog_tracing_options(true, need_memory, false), + ); + + let memory_structlog = normalize_structlog(&memory_structlog_fut.await?).await; + + Ok::>, RpcError>(memory_structlog) + }; - match timeout(TIMEOUT_LIMIT, structlog).await { + match timeout(TIMEOUT_LIMIT, inner).await { Err(ellapsed_error) => Err(RpcError::Transport(TransportErrorKind::Custom(Box::new( ellapsed_error, )))), - Ok(structlog_res) => Ok(normalize_structlog(structlog_res?).await), + Ok(structlog_res) => Ok(structlog_res?), } } @@ -470,11 +482,11 @@ pub mod structlogprime { } pub(crate) async fn normalize_structlog( - unnormalized_structlog: GethTrace, + unnormalized_structlog: &GethTrace, ) -> Option> { match unnormalized_structlog { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs), - GethTrace::JS(structlog_js_object) => try_reserialize(&structlog_js_object) + GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs.clone()), + GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) .ok() .map(|s| s.struct_logs), _ => None, From 8bce013d66407b8b097274328dbd0998493d9977 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 18:54:58 +0200 Subject: [PATCH 022/112] avoid ensure macro --- zero/src/rpc/jumpdest.rs | 181 +++++++++++++++++++++------------------ 1 file changed, 99 insertions(+), 82 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index f44aa7a43..d916ba4b5 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -176,7 +176,10 @@ pub(crate) fn generate_jumpdest_table( call_stack.pop(); } - ensure!(call_stack.is_empty().not(), "Call stack was empty."); + ensure!( + call_stack.is_empty().not(), + "Call stack was unexpectedly empty." + ); let (code_hash, ctx) = call_stack.last().unwrap(); trace!("TX: {:?}", tx.hash); @@ -190,26 +193,32 @@ pub(crate) fn generate_jumpdest_table( match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands = 2; // actually 6 or 7. - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); + let operands_used = 2; // actually 6 or 7. + + if evm_stack.len() < operands_used { + trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len()); + // maybe increment ctx here + continue; + } // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 let [_gas, address, ..] = evm_stack[..] else { unreachable!() }; - let callee_address = { - let callee_raw = *address; - ensure!(callee_raw <= U256::from(U160::MAX)); - let lower_20_bytes = U160::from(callee_raw); - Address::from(lower_20_bytes) + if *address > U256::from(U160::MAX) { + trace!( + "{op}: Callee address {} was larger than possible {}.", + *address, + U256::from(U160::MAX) + ); + continue; }; + let lower_20_bytes = U160::from(*address); + let callee_address = Address::from(lower_20_bytes); if callee_addr_to_code_hash.contains_key(&callee_address) { let next_code_hash = callee_addr_to_code_hash[&callee_address]; @@ -231,69 +240,63 @@ pub(crate) fn generate_jumpdest_table( ); } next_ctx_available += 1; - prev_jump = None; } - "CREATE" => { + "CREATE" | "CREATE2" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands = 3; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); - let [_value, _offset, _size, ..] = evm_stack[..] else { - unreachable!() - }; + let operands_used = 3; - let contract_address = tx.from.create(tx.nonce); - ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); - let code_hash = callee_addr_to_code_hash[&contract_address]; - call_stack.push((code_hash, next_ctx_available)); + if evm_stack.len() < operands_used { + trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len() ); + continue; + }; - next_ctx_available += 1; - prev_jump = None; - } - "CREATE2" => { - ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); - // We reverse the stack, so the order matches our assembly code. - let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands = 4; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); - let [_value, offset, size, _salt, ..] = evm_stack[..] else { + let [_value, offset, size, ..] = evm_stack[..] else { unreachable!() }; - ensure!(*offset <= U256::from(usize::MAX)); + if *offset > U256::from(usize::MAX) { + trace!( + "{op}: Offset {offset} was too large to fit in usize {}.", + usize::MAX + ); + continue; + }; let offset: usize = offset.to(); - ensure!(*size <= U256::from(usize::MAX)); + if *size > U256::from(usize::MAX) { + trace!( + "{op}: Size {size} was too large to fit in usize {}.", + usize::MAX + ); + continue; + }; let size: usize = size.to(); + let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; - // let salt: Word = salt.to_be_bytes(); - ensure!( - entry.memory.is_some() && offset + size <= memory_size, - "Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size - ); + if entry.memory.is_none() || offset + size > memory_size { + trace!("Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size); + continue; + } let memory_raw: &[String] = entry.memory.as_ref().unwrap(); let memory_parsed: Vec> = memory_raw .iter() - .map(|s| { - // let c = s.parse(); - let c = U256::from_str_radix(s, 16); - ensure!(c.is_ok(), "Parsing memory failed."); - let a: U256 = c.unwrap(); - let d: Word = a.to_be_bytes(); - Ok(d) + .map(|mem_line| { + let mem_line_parsed = U256::from_str_radix(mem_line, 16)?; + Ok(mem_line_parsed.to_be_bytes()) }) .collect(); let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); - let memory: Vec = mem_res?.concat(); + if mem_res.is_err() { + trace!( + "{op}: Parsing memory failed with error: {}", + mem_res.unwrap_err() + ); + continue; + } + let memory: Vec = mem_res.unwrap().concat(); let init_code = &memory[offset..offset + size]; let init_code_hash = keccak(init_code); @@ -303,18 +306,17 @@ pub(crate) fn generate_jumpdest_table( call_stack.push((init_code_hash, next_ctx_available)); next_ctx_available += 1; - prev_jump = None; } "JUMP" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); let operands = 1; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); + if evm_stack.len() < operands { + trace!( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); + continue; + } let [jump_target, ..] = evm_stack[..] else { unreachable!() }; @@ -322,42 +324,55 @@ pub(crate) fn generate_jumpdest_table( prev_jump = Some(*jump_target); } "JUMPI" => { + prev_jump = None; ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); let operands = 2; - ensure!( - evm_stack.len() >= operands, - "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", - evm_stack.len() - ); + if evm_stack.len() < operands { + trace!( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len()); + continue; + }; + let [jump_target, condition, ..] = evm_stack[..] else { unreachable!() }; let jump_condition = condition.is_zero().not(); - prev_jump = if jump_condition { - Some(*jump_target) - } else { - None - }; + if jump_condition { + prev_jump = Some(*jump_target) + } } "JUMPDEST" => { - let jumped_here = if let Some(jmp_target) = prev_jump { - jmp_target == U256::from(entry.pc) - } else { - false - }; - let jumpdest_offset = TryInto::::try_into(entry.pc)?; - if jumped_here { - jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset); + let mut jumped_here = false; + + if let Some(jmp_target) = prev_jump { + jumped_here = jmp_target == U256::from(entry.pc); } - // else: we do not care about JUMPDESTs reached through fall-through. prev_jump = None; + + if jumped_here.not() { + trace!( + "{op}: JUMPDESTs at offset {} was reached through fall-through.", + entry.pc + ); + continue; + } + + let jumpdest_offset = TryInto::::try_into(entry.pc); + if jumpdest_offset.is_err() { + trace!( + "{op}: Could not cast offset {} to usize {}.", + entry.pc, + usize::MAX + ); + continue; + } + jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset.unwrap()); } "EXTCODECOPY" | "EXTCODESIZE" => { - next_ctx_available += 1; prev_jump = None; + next_ctx_available += 1; } _ => { prev_jump = None; @@ -367,6 +382,8 @@ pub(crate) fn generate_jumpdest_table( Ok(jumpdest_table) } +/// This module exists as a workaround for parsing `StrucLog`. The `error` +/// field is a string in Alloy but an object in Erigon. pub mod structlogprime { use core::option::Option::None; use std::collections::BTreeMap; From b0ebc2c2fe0999940ed52f105a4df0c43d150232 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 20 Sep 2024 19:25:33 +0200 Subject: [PATCH 023/112] fix CREATE --- zero/src/rpc/jumpdest.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index d916ba4b5..8ef51bbfa 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -52,7 +52,9 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } fn trace_contains_create2(structlog: &[StructLog]) -> bool { - structlog.iter().any(|entry| entry.op == "CREATE2") + structlog + .iter() + .any(|entry| entry.op == "CREATE" || entry.op == "CREATE2") } // Gets the lightest possible structlog for transcation `tx_hash`. From 62c7053d36a16a67287a70fb370773cb66a091e2 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 23 Sep 2024 11:27:07 +0200 Subject: [PATCH 024/112] small adjustments --- Cargo.lock | 1 - Cargo.toml | 2 + evm_arithmetization/Cargo.toml | 1 - .../src/generation/prover_input.rs | 2 +- scripts/test_native.sh | 51 ++++++++++++------- zero/src/rpc/jumpdest.rs | 6 +-- 6 files changed, 39 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f929a38c1..0ed81ea1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2068,7 +2068,6 @@ dependencies = [ "plonky2", "plonky2_maybe_rayon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "plonky2_util", - "primitive-types 0.12.2", "rand", "rand_chacha", "ripemd", diff --git a/Cargo.toml b/Cargo.toml index 386ecff5e..cac073af2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ criterion = "0.5.1" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" +enumn = "0.1.13" env_logger = "0.11.3" eth_trie = "0.4.0" ethereum-types = "0.14.1" @@ -89,6 +90,7 @@ serde = "1.0.203" serde-big-array = "0.5.1" serde_json = "1.0.118" serde_path_to_error = "0.1.16" +serde_with = "3.8.1" sha2 = "0.10.8" static_assertions = "1.1.0" thiserror = "1.0.61" diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index e8e832c2d..ffdb8d2f5 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -15,7 +15,6 @@ homepage.workspace = true keywords.workspace = true [dependencies] -__compat_primitive_types = { workspace = true } anyhow = { workspace = true } bytes = { workspace = true } env_logger = { workspace = true } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index e30ea3b6e..b502b68af 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -767,7 +767,7 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - dbg!(&self.inputs.jumpdest_table); + // dbg!(&self.inputs.jumpdest_table); // w for witness // let txn_idx = self.next_txn_index - 1; // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code diff --git a/scripts/test_native.sh b/scripts/test_native.sh index d2a7aac5f..943a00ca3 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -10,22 +10,25 @@ fi mkdir -p witnesses # Must match the values in prove_stdio.sh or build is dirty. -export RAYON_NUM_THREADS=4 -export TOKIO_WORKER_THREADS=4 +export RAYON_NUM_THREADS=1 +export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full -export RUST_LOG=info -export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' -export RUST_MIN_STACK=33554432 +#export RUST_LOG=info +#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' +#export RUST_MIN_STACK=33554432 -MAINNETBLOCKS=" +CANCUNBLOCKS=" 20548415 20240058 19665756 20634472 19807080 20634403 +" + +PRECANCUN=" 19096840 19240700 " @@ -33,17 +36,18 @@ MAINNETBLOCKS=" CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` STATICTIP=20721266 -NUMRANDOMBLOCKS=10 +NUMRANDOMBLOCKS=100 RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." -#BLOCKS="$MAINNETBLOCKS $RANDOMBLOCKS" -BLOCKS="$MAINNETBLOCKS" +BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" +#BLOCKS="$CANCUNBLOCKS" echo "Testing blocks: $BLOCKS" echo "Downloading witnesses.." +echo "------------------------"| tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" @@ -56,20 +60,15 @@ for BLOCK in $BLOCKS; do then printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt break + else + printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure | tee -a witnesses/native_results.txt fi - - printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure| tee -a witnesses/native_results.txt done echo "Witness for block $BLOCK ($WITNESS) prepared." -done - -echo "Finished downloading witnesses." -echo "Testing prepared witnesses.." -for WITNESS in witnesses/*.native.$GITHASH.witness.json; do echo "Testing $WITNESS" - prove_stdio.sh $WITNESS test_only + ./prove_stdio.sh $WITNESS test_only EXITCODE=$? if [ $EXITCODE -eq 0 ] then @@ -80,4 +79,20 @@ for WITNESS in witnesses/*.native.$GITHASH.witness.json; do printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt done -echo "Finished testing witnesses." +#echo "Finished downloading witnesses." +#echo "Testing prepared witnesses.." +# +#for WITNESS in witnesses/*.native.$GITHASH.witness.json; do +# echo "Testing $WITNESS" +# ./prove_stdio.sh $WITNESS test_only +# EXITCODE=$? +# if [ $EXITCODE -eq 0 ] +# then +# RESULT="success" +# else +# RESULT="failure" +# fi +# printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt +#done +# +#echo "Finished testing witnesses." diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 8ef51bbfa..38894679a 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -30,7 +30,7 @@ use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(10); +const TIMEOUT_LIMIT: Duration = Duration::from_secs(2*60); /// Structure of Etheruem memory type Word = [u8; 32]; @@ -51,7 +51,7 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } -fn trace_contains_create2(structlog: &[StructLog]) -> bool { +fn trace_contains_create(structlog: &[StructLog]) -> bool { structlog .iter() .any(|entry| entry.op == "CREATE" || entry.op == "CREATE2") @@ -77,7 +77,7 @@ where let need_memory = stackonly_structlog_opt .as_deref() - .is_some_and(trace_contains_create2); + .is_some_and(trace_contains_create); trace!("Need structlog with memory: {need_memory}"); if need_memory.not() { From 74b86fd916f6ab41fbec9cbe4e69dc16448be3de Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 23 Sep 2024 13:39:22 +0200 Subject: [PATCH 025/112] fmt --- zero/src/rpc/jumpdest.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 38894679a..d9ba8cd4c 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -30,7 +30,7 @@ use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(2*60); +const TIMEOUT_LIMIT: Duration = Duration::from_secs(2 * 60); /// Structure of Etheruem memory type Word = [u8; 32]; @@ -51,13 +51,15 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } +/// Predicate that determines whether a `StructLog` that includes memory is +/// required. fn trace_contains_create(structlog: &[StructLog]) -> bool { structlog .iter() .any(|entry| entry.op == "CREATE" || entry.op == "CREATE2") } -// Gets the lightest possible structlog for transcation `tx_hash`. +/// Gets the lightest possible structlog for transcation `tx_hash`. pub(crate) async fn get_normalized_structlog( provider: &ProviderT, tx_hash: &B256, From c8be8888fb190cdcfa5abcaff04f0bfbf575684b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 23 Sep 2024 14:29:46 +0200 Subject: [PATCH 026/112] feedback --- scripts/test_native.sh | 2 +- zero/src/prover/cli.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 943a00ca3..56de59a49 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -uxo pipefail +set -uo pipefail export RPC= if [ -z $RPC ]; then diff --git a/zero/src/prover/cli.rs b/zero/src/prover/cli.rs index 0dfa383b6..a6cdaebf9 100644 --- a/zero/src/prover/cli.rs +++ b/zero/src/prover/cli.rs @@ -18,7 +18,7 @@ pub struct CliProverConfig { #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 19)] max_cpu_len_log: usize, /// Number of transactions in a batch to process at once. - #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 1)] + #[arg(short, long, help_heading = HELP_HEADING, default_value_t = 10)] batch_size: usize, /// If true, save the public inputs to disk on error. #[arg(short='i', long, help_heading = HELP_HEADING, default_value_t = false)] From d00439fc64fa2c9eb88aa58228ea007bf4e03e53 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 12:20:12 +0200 Subject: [PATCH 027/112] feedback --- .gitignore | 1 - zero/src/rpc/jumpdest.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 9f5ee7a64..0016ff3b4 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,3 @@ ################# /proofs /**/*.zkproof -/witnesses diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index d9ba8cd4c..558804bc7 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -386,7 +386,7 @@ pub(crate) fn generate_jumpdest_table( Ok(jumpdest_table) } -/// This module exists as a workaround for parsing `StrucLog`. The `error` +/// This module exists as a workaround for parsing `StructLog`. The `error` /// field is a string in Alloy but an object in Erigon. pub mod structlogprime { use core::option::Option::None; From 6876c07e9b1f07b7afb2e673288834fc093e6355 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 16:31:51 +0200 Subject: [PATCH 028/112] Add JumpdestSrc parameter --- trace_decoder/src/core.rs | 4 ++-- zero/src/bin/rpc.rs | 4 ++++ zero/src/rpc/mod.rs | 8 ++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index cd410e58b..78c9d9b96 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -6,10 +6,9 @@ use std::{ mem, }; -use alloy::primitives::address; use alloy::{ consensus::{Transaction, TxEnvelope}, - primitives::TxKind, + primitives::{address, TxKind}, rlp::Decodable as _, }; use alloy_compat::Compat as _; @@ -138,6 +137,7 @@ pub fn entrypoint( block_hashes: b_hashes.clone(), burn_addr, jumpdest_table: { + // TODO See the issue Simulate to get jumpdests on a per-transaction basis #653. // Note that this causes any batch containing just a single `None` to collapse // into a `None`, which causing failover to simulating jumpdest analysis for the // whole batch. There is an optimization opportunity here. diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index b9751196c..115a7619e 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -15,6 +15,7 @@ use zero::block_interval::BlockIntervalStream; use zero::prover::BlockProverInput; use zero::provider::CachedProvider; use zero::rpc; +use zero::rpc::JumpdestSrc; use self::rpc::{retry::build_http_retry_provider, RpcType}; @@ -34,6 +35,9 @@ struct RpcToolConfig { /// The RPC Tracer Type. #[arg(short = 't', long, default_value = "jerigon")] rpc_type: RpcType, + /// The source of jumpdest tables. + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + jumpdest_src: JumpdestSrc, /// Backoff in milliseconds for retry requests. #[arg(long, default_value_t = 0)] backoff: u64, diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index e3c218eb6..da1ac474b 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -39,6 +39,14 @@ pub enum RpcType { Native, } +/// The Jumpdest source type. +#[derive(ValueEnum, Clone, Debug, Copy)] +pub enum JumpdestSrc { + Simulation, + Zero, + Jerigon, +} + /// Obtain the prover input for one block pub async fn block_prover_input( cached_provider: Arc>, From 60efef9383b45954c9a735e5a9ae669fa89d43e7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 17:36:09 +0200 Subject: [PATCH 029/112] Refactor --- zero/src/bin/leader.rs | 2 ++ zero/src/bin/leader/cli.rs | 5 ++++- zero/src/bin/leader/client.rs | 4 +++- zero/src/bin/rpc.rs | 4 ++++ zero/src/rpc/jerigon.rs | 36 ++++++++++++++++++++--------------- zero/src/rpc/mod.rs | 17 +++++++++++++++-- zero/src/rpc/native/mod.rs | 3 +++ 7 files changed, 52 insertions(+), 19 deletions(-) diff --git a/zero/src/bin/leader.rs b/zero/src/bin/leader.rs index 728f6bb0e..775439fde 100644 --- a/zero/src/bin/leader.rs +++ b/zero/src/bin/leader.rs @@ -72,6 +72,7 @@ async fn main() -> Result<()> { Command::Rpc { rpc_url, rpc_type, + jumpdest_src, block_interval, checkpoint_block_number, previous_proof, @@ -91,6 +92,7 @@ async fn main() -> Result<()> { backoff, max_retries, block_time, + jumpdest_src, }, block_interval, LeaderConfig { diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index 3569efc41..78ad0fbdb 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -4,7 +4,7 @@ use alloy::transports::http::reqwest::Url; use clap::{Parser, Subcommand, ValueHint}; use zero::prover::cli::CliProverConfig; use zero::prover_state::cli::CliProverStateConfig; -use zero::rpc::RpcType; +use zero::rpc::{JumpdestSrc, RpcType}; /// zero-bin leader config #[derive(Parser)] @@ -43,6 +43,9 @@ pub(crate) enum Command { // The node RPC type (jerigon / native). #[arg(long, short = 't', default_value = "jerigon")] rpc_type: RpcType, + /// The source of jumpdest tables. + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + jumpdest_src: JumpdestSrc, /// The block interval for which to generate a proof. #[arg(long, short = 'i')] block_interval: String, diff --git a/zero/src/bin/leader/client.rs b/zero/src/bin/leader/client.rs index 6825b6683..136fb0845 100644 --- a/zero/src/bin/leader/client.rs +++ b/zero/src/bin/leader/client.rs @@ -10,7 +10,7 @@ use tracing::info; use zero::block_interval::{BlockInterval, BlockIntervalStream}; use zero::pre_checks::check_previous_proof_and_checkpoint; use zero::prover::{self, BlockProverInput, ProverConfig}; -use zero::rpc; +use zero::rpc::{self, JumpdestSrc}; use zero::rpc::{retry::build_http_retry_provider, RpcType}; #[derive(Debug)] @@ -20,6 +20,7 @@ pub struct RpcParams { pub backoff: u64, pub max_retries: u32, pub block_time: u64, + pub jumpdest_src: JumpdestSrc, } #[derive(Debug)] @@ -87,6 +88,7 @@ pub(crate) async fn client_main( block_id, leader_config.checkpoint_block_number, rpc_params.rpc_type, + rpc_params.jumpdest_src, ) .await?; block_tx diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index 115a7619e..3c5c3ed88 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -25,6 +25,7 @@ struct FetchParams { pub end_block: u64, pub checkpoint_block_number: Option, pub rpc_type: RpcType, + pub jumpdest_src: JumpdestSrc, } #[derive(Args, Clone, Debug)] @@ -105,6 +106,7 @@ where block_id, checkpoint_block_number, params.rpc_type, + params.jumpdest_src, ) .await?; @@ -133,6 +135,7 @@ impl Cli { end_block, checkpoint_block_number, rpc_type: self.config.rpc_type, + jumpdest_src: self.config.jumpdest_src, }; let block_prover_inputs = @@ -159,6 +162,7 @@ impl Cli { end_block: block_number, checkpoint_block_number: None, rpc_type: self.config.rpc_type, + jumpdest_src: self.config.jumpdest_src, }; let block_prover_inputs = diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index ab8cf8d37..15ad0edf1 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -1,6 +1,8 @@ +use core::iter::Iterator; use std::collections::BTreeMap; use std::ops::Deref as _; +use __compat_primitive_types::H160; use alloy::{ providers::Provider, rpc::types::{eth::BlockId, trace::geth::StructLog, Block, BlockTransactionsKind, Transaction}, @@ -21,6 +23,7 @@ use tracing::info; use super::{ fetch_other_block_data, jumpdest::{self, get_normalized_structlog}, + JumpdestSrc, }; use crate::prover::BlockProverInput; use crate::provider::CachedProvider; @@ -36,6 +39,7 @@ pub async fn block_prover_input( cached_provider: std::sync::Arc>, target_block_id: BlockId, checkpoint_block_number: u64, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result where ProviderT: Provider, @@ -65,17 +69,18 @@ where .get_block(target_block_id, BlockTransactionsKind::Full) .await?; - let tx_traces: Vec<&BTreeMap<__compat_primitive_types::H160, TxnTrace>> = tx_results - .iter() - .map(|TxnInfo { traces, meta: _ }| traces) - .collect::>(); - - let jdts: Vec> = process_transactions( - &block, - cached_provider.get_provider().await?.deref(), - &tx_traces, - ) - .await?; + let jdts: Vec> = match jumpdest_src { + JumpdestSrc::Simulation => vec![None; tx_results.len()], + JumpdestSrc::Zero => { + process_transactions( + &block, + cached_provider.get_provider().await?.deref(), + tx_results.iter().map(|TxnInfo { traces, meta: _ }| traces), // &tx_traces, + ) + .await? + } + JumpdestSrc::Jerigon => todo!(), + }; // weave in the JDTs let txn_info = tx_results @@ -124,14 +129,15 @@ where } /// Processes the transactions in the given block and updates the code db. -pub async fn process_transactions( +pub async fn process_transactions<'i, I, ProviderT, TransportT>( block: &Block, provider: &ProviderT, - tx_traces: &[&BTreeMap<__compat_primitive_types::H160, TxnTrace>], + tx_traces: I, ) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, + I: Iterator>, { let futures = block .transactions @@ -139,7 +145,7 @@ where .context("No transactions in block")? .iter() .zip(tx_traces) - .map(|(tx, &tx_trace)| process_transaction(provider, tx, tx_trace)) + .map(|(tx, tx_trace)| process_transaction(provider, tx, tx_trace)) .collect::>(); let vec_of_res = futures.collect::>().await; @@ -151,7 +157,7 @@ where pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, - tx_trace: &BTreeMap<__compat_primitive_types::H160, TxnTrace>, + tx_trace: &BTreeMap, ) -> anyhow::Result> where ProviderT: Provider, diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index da1ac474b..061cd9743 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -53,6 +53,7 @@ pub async fn block_prover_input( block_id: BlockId, checkpoint_block_number: u64, rpc_type: RpcType, + jumpdest_src: JumpdestSrc, ) -> Result where ProviderT: Provider, @@ -60,10 +61,22 @@ where { match rpc_type { RpcType::Jerigon => { - jerigon::block_prover_input(cached_provider, block_id, checkpoint_block_number).await + jerigon::block_prover_input( + cached_provider, + block_id, + checkpoint_block_number, + jumpdest_src, + ) + .await } RpcType::Native => { - native::block_prover_input(cached_provider, block_id, checkpoint_block_number).await + native::block_prover_input( + cached_provider, + block_id, + checkpoint_block_number, + jumpdest_src, + ) + .await } } } diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 5563cad5b..6dcf07af3 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -17,11 +17,14 @@ mod txn; pub use txn::{process_transaction, process_transactions}; +use super::JumpdestSrc; + /// Fetches the prover input for the given BlockId. pub async fn block_prover_input( provider: Arc>, block_number: BlockId, checkpoint_block_number: u64, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result where ProviderT: Provider, From b07752dd60a45b626c0fc168661ba0cee76640e9 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 18:12:43 +0200 Subject: [PATCH 030/112] Add jmp src to native --- zero/src/rpc/native/mod.rs | 11 ++++++++--- zero/src/rpc/native/txn.rs | 39 +++++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 6dcf07af3..76dd302ae 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -31,7 +31,7 @@ where TransportT: Transport + Clone, { let (block_trace, other_data) = try_join!( - process_block_trace(provider.clone(), block_number), + process_block_trace(provider.clone(), block_number, jumpdest_src), crate::rpc::fetch_other_block_data(provider.clone(), block_number, checkpoint_block_number) )?; @@ -45,6 +45,7 @@ where pub(crate) async fn process_block_trace( cached_provider: Arc>, block_number: BlockId, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result where ProviderT: Provider, @@ -54,8 +55,12 @@ where .get_block(block_number, BlockTransactionsKind::Full) .await?; - let (code_db, txn_info) = - txn::process_transactions(&block, cached_provider.get_provider().await?.deref()).await?; + let (code_db, txn_info) = txn::process_transactions( + &block, + cached_provider.get_provider().await?.deref(), + jumpdest_src, + ) + .await?; let trie_pre_images = state::process_state_witness(cached_provider, block, &txn_info).await?; Ok(BlockTrace { diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 34ed820df..582b7ee8f 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -1,3 +1,4 @@ +use core::option::Option::None; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use __compat_primitive_types::{H256, U256}; @@ -24,13 +25,17 @@ use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use tracing::info; -use crate::rpc::jumpdest::{self, get_normalized_structlog}; use crate::rpc::Compat; +use crate::rpc::{ + jumpdest::{self, get_normalized_structlog}, + JumpdestSrc, +}; /// Processes the transactions in the given block and updates the code db. pub async fn process_transactions( block: &Block, provider: &ProviderT, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result<(CodeDb, Vec)> where ProviderT: Provider, @@ -41,7 +46,7 @@ where .as_transactions() .context("No transactions in block")? .iter() - .map(|tx| process_transaction(provider, tx)) + .map(|tx| process_transaction(provider, tx, jumpdest_src)) .collect::>() .try_fold( (BTreeSet::new(), Vec::new()), @@ -59,13 +64,14 @@ where pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result<(CodeDb, TxnInfo)> where ProviderT: Provider, TransportT: Transport + Clone, { let (tx_receipt, pre_trace, diff_trace, structlog_opt) = - fetch_tx_data(provider, &tx.hash).await?; + fetch_tx_data(provider, &tx.hash, jumpdest_src).await?; let tx_status = tx_receipt.status(); let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); @@ -125,6 +131,7 @@ where async fn fetch_tx_data( provider: &ProviderT, tx_hash: &B256, + jumpdest_src: JumpdestSrc, ) -> anyhow::Result<( ::ReceiptResponse, GethTrace, @@ -138,14 +145,24 @@ where let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); - let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); - - let (tx_receipt, pre_trace, diff_trace, structlog_trace) = futures::try_join!( - tx_receipt_fut, - pre_trace_fut, - diff_trace_fut, - structlog_trace_fut, - )?; + + let (tx_receipt, pre_trace, diff_trace, structlog_trace) = match jumpdest_src { + JumpdestSrc::Zero => { + let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); + futures::try_join!( + tx_receipt_fut, + pre_trace_fut, + diff_trace_fut, + structlog_trace_fut, + )? + } + JumpdestSrc::Simulation => { + let (tx_receipt, pre_trace, diff_trace) = + futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; + (tx_receipt, pre_trace, diff_trace, None) + } + JumpdestSrc::Jerigon => todo!(), + }; Ok(( tx_receipt.context("Transaction receipt not found.")?, From 66ea81151d7b12d4dfedb846eb3d2e0db560ef16 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 20:06:22 +0200 Subject: [PATCH 031/112] Feedback --- trace_decoder/src/core.rs | 9 +++++---- trace_decoder/src/lib.rs | 1 + zero/src/rpc/jumpdest.rs | 14 ++++++++------ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 78c9d9b96..4890a68b4 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -34,6 +34,11 @@ use crate::{ SeparateTriePreImages, TxnInfo, TxnMeta, TxnTrace, }; +/// Addresses of precompiled Ethereum contracts. +pub const PRECOMPILE_ADDRESSES: Range = + address!("0000000000000000000000000000000000000001") + ..address!("000000000000000000000000000000000000000a"); + /// TODO(0xaatif): document this after https://github.com/0xPolygonZero/zk_evm/issues/275 pub fn entrypoint( trace: BlockTrace, @@ -496,10 +501,6 @@ fn middle( state_mask.insert(TrieKey::from_address(addr)); } else { // Simple state access - const PRECOMPILE_ADDRESSES: Range = - address!("0000000000000000000000000000000000000001") - ..address!("000000000000000000000000000000000000000a"); - if receipt.status || !PRECOMPILE_ADDRESSES.contains(&addr.compat()) { // TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/pull/613 // masking like this SHOULD be a space-saving optimization, diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs index 8c54eea8e..587707883 100644 --- a/trace_decoder/src/lib.rs +++ b/trace_decoder/src/lib.rs @@ -67,6 +67,7 @@ mod typed_mpt; mod wire; pub use core::entrypoint; +pub use core::PRECOMPILE_ADDRESSES; mod core; diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 558804bc7..fc42ebb11 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -26,6 +26,7 @@ use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::TxnTrace; +use trace_decoder::PRECOMPILE_ADDRESSES; use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over @@ -145,7 +146,9 @@ pub(crate) fn generate_jumpdest_table( ); let entrypoint_code_hash: H256 = match tx.to { - Some(to_address) if precompiles().contains(&to_address) => return Ok(jumpdest_table), + Some(to_address) if PRECOMPILE_ADDRESSES.contains(&to_address) => { + return Ok(jumpdest_table) + } Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { return Ok(jumpdest_table) } @@ -201,11 +204,13 @@ pub(crate) fn generate_jumpdest_table( ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); // We reverse the stack, so the order matches our assembly code. let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands_used = 2; // actually 6 or 7. + // These opcodes expect 6 or 7 operands on the stack, but for jumpdest-table + // generation we only use 2, and failures will be handled in + // next iteration by popping the stack accordingly. + let operands_used = 2; if evm_stack.len() < operands_used { trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len()); - // maybe increment ctx here continue; } // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 @@ -304,9 +309,6 @@ pub(crate) fn generate_jumpdest_table( let init_code = &memory[offset..offset + size]; let init_code_hash = keccak(init_code); - // let contract_address = tx.from.create2_from_code(salt, init_code); - // ensure!(callee_addr_to_code_hash.contains_key(&contract_address)); - // let code_hash = callee_addr_to_code_hash[&contract_address]; call_stack.push((init_code_hash, next_ctx_available)); next_ctx_available += 1; From f230b84423df8f268d4d56a98bc2e4042bf7649d Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 24 Sep 2024 20:16:24 +0200 Subject: [PATCH 032/112] fixup! Feedback --- zero/src/rpc/jumpdest.rs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index fc42ebb11..238ac7b7a 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -3,9 +3,7 @@ use core::option::Option::None; use core::time::Duration; use std::collections::BTreeMap; use std::collections::HashMap; -use std::collections::HashSet; use std::ops::Not as _; -use std::sync::OnceLock; use __compat_primitive_types::H256; use alloy::primitives::Address; @@ -105,15 +103,6 @@ where } } -/// Provides a way to check in constant time if an address points to a -/// precompile. -fn precompiles() -> &'static HashSet
{ - static PRECOMPILES: OnceLock> = OnceLock::new(); - PRECOMPILES.get_or_init(|| { - HashSet::
::from_iter((1..=0xa).map(|x| Address::from(U160::from(x)))) - }) -} - /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. pub(crate) fn generate_jumpdest_table( @@ -234,12 +223,12 @@ pub(crate) fn generate_jumpdest_table( call_stack.push((next_code_hash, next_ctx_available)); }; - if precompiles().contains(&callee_address) { + if PRECOMPILE_ADDRESSES.contains(&callee_address) { trace!("Called precompile at address {}.", &callee_address); }; if callee_addr_to_code_hash.contains_key(&callee_address).not() - && precompiles().contains(&callee_address).not() + && PRECOMPILE_ADDRESSES.contains(&callee_address).not() { // This case happens if calling an EOA. This is described // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun From 90722a30578b7e3aac2d5b107fae46ebb79e6f3c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 25 Sep 2024 23:29:58 +0200 Subject: [PATCH 033/112] feedback --- zero/src/rpc/jerigon.rs | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 15ad0edf1..5fcf7f584 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -79,36 +79,21 @@ where ) .await? } - JumpdestSrc::Jerigon => todo!(), + JumpdestSrc::Jerigon => todo!("hybrid server bulk struct log retrieval/local jumpdest table generation not yet implemented"), }; + let mut code_db = CodeDb::default(); // weave in the JDTs let txn_info = tx_results .into_iter() .zip(jdts) - .map( - |( - TxnInfo { - traces, - meta: - TxnMeta { - byte_code, - new_receipt_trie_node_byte, - gas_used, - jumpdest_table: _, - }, - }, - jdt, - )| TxnInfo { - traces, - meta: TxnMeta { - byte_code, - new_receipt_trie_node_byte, - gas_used, - jumpdest_table: jdt, - }, - }, - ) + .map(|(mut tx_info, jdt)| { + tx_info.meta.jumpdest_table = jdt.map(|(j, c)| { + code_db.extend(c); + j + }); + tx_info + }) .collect(); let other_data = @@ -148,8 +133,11 @@ where .map(|(tx, tx_trace)| process_transaction(provider, tx, tx_trace)) .collect::>(); - let vec_of_res = futures.collect::>().await; - vec_of_res.into_iter().collect::, _>>() + futures + .collect::>() + .await + .into_iter() + .collect::, _>>() } /// Processes the transaction with the given transaction hash and updates the From a0e0879d74624adb1c0bab36e62c2d78bc34da0e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 25 Sep 2024 23:33:07 +0200 Subject: [PATCH 034/112] fix missing code for CREATE --- .../src/cpu/kernel/interpreter.rs | 5 +- .../src/generation/prover_input.rs | 11 +-- scripts/test_jerigon.sh | 86 ++++++++++++++++++- trace_decoder/src/core.rs | 22 ++++- zero/src/rpc/jerigon.rs | 25 +++--- zero/src/rpc/jumpdest.rs | 11 ++- zero/src/rpc/native/txn.rs | 18 ++-- 7 files changed, 143 insertions(+), 35 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 9c2f7bfac..5ab8fc659 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -173,9 +173,10 @@ pub(crate) fn set_jumpdest_analysis_inputs_rpc( &vec![] }; trace!( - "code: {:?}, code_addr: {:?} <============", + "code: {:?}, code_addr: {:?}, {:?} <============", &code, - &code_addr + &code_addr, + code_map.contains_key(code_addr), ); trace!("code_map: {:?}", &code_map); prove_context_jumpdests(code, ctx_jumpdests) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index b502b68af..b7d4a6788 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -767,23 +767,20 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - // dbg!(&self.inputs.jumpdest_table); - // w for witness - // let txn_idx = self.next_txn_index - 1; - // let rpcw = self.inputs.jumpdest_tables[txn_idx].as_ref();contract_code + dbg!(&self.inputs.jumpdest_table); + dbg!(&self.inputs.txn_hashes); let rpcw = &self.inputs.jumpdest_table; let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + info!("Generating JUMPDEST tables part2"); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); let (sim, simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) - && rw != &sw - { + if let (Some(rw), Some(sw)) = (rpcw, simw) { info!("SIMW {}", sw); info!("RPCW {}", rw); assert_eq!(rw, &sw); diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 6d1125216..5426b5b01 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -153,6 +153,89 @@ NOWSUCCESS=" 35 " +ROUND3=" +125 +127 +131 +132 +136 +141 +142 +143 +145 +149 +150 +151 +153 +154 +186 +187 +188 +190 +193 +195 +197 +199 +201 +214 +220 +221 +222 +223 +226 +228 +229 +230 +231 +232 +234 +242 +256 +257 +258 +262 +264 +267 +268 +282 +284 +285 +287 +292 +294 +295 +301 +303 +304 +321 +325 +333 +460 +461 +462 +463 +464 +465 +466 +467 +468 +473 +474 +528 +529 +530 +531 +532 +533 +534 +566 +570 +664 +77 +548 +" + + # 470..663 from Robin for i in {470..663} do @@ -163,7 +246,8 @@ TIP=688 NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` -BLOCKS="$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS" +#$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3" +BLOCKS="$ROUND3" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 4890a68b4..c6e530a75 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -132,11 +132,24 @@ pub fn entrypoint( } }); - contract_code + let initmap: HashMap<_, _> = initcodes .into_iter() - .chain(initcodes) .map(|it| (keccak_hash::keccak(&it), it)) - .collect() + .collect(); + + let contractmap: HashMap<_, _> = contract_code + .into_iter() + .map(|it| (keccak_hash::keccak(&it), it)) + .collect(); + + log::trace!("Initmap {:?}", initmap); + log::trace!("Contractmap {:?}", contractmap); + //log::trace!("DECODER: {:#?}", res); + + let mut c = code.inner.clone(); + c.extend(contractmap); + c.extend(initmap); + c }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), @@ -643,9 +656,10 @@ fn map_receipt_bytes(bytes: Vec) -> anyhow::Result> { /// trace. /// If there are any txns that create contracts, then they will also /// get added here as we process the deltas. +#[derive(Clone)] struct Hash2Code { /// Key must always be [`hash`] of value. - inner: HashMap>, + pub inner: HashMap>, } impl Hash2Code { diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 5fcf7f584..0b79cfd70 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -10,14 +10,12 @@ use alloy::{ }; use alloy_primitives::Address; use anyhow::Context as _; -use evm_arithmetization::jumpdest::JumpDestTableWitness; +use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::FuturesOrdered; use futures::StreamExt as _; use serde::Deserialize; use serde_json::json; -use trace_decoder::{ - BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnMeta, TxnTrace, -}; +use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnTrace}; use tracing::info; use super::{ @@ -69,7 +67,7 @@ where .get_block(target_block_id, BlockTransactionsKind::Full) .await?; - let jdts: Vec> = match jumpdest_src { + let jdts: Vec> = match jumpdest_src { JumpdestSrc::Simulation => vec![None; tx_results.len()], JumpdestSrc::Zero => { process_transactions( @@ -107,7 +105,7 @@ where .context("invalid hex returned from call to eth_getWitness")?, }), txn_info, - code_db: Default::default(), + code_db, }, other_data, }) @@ -118,7 +116,7 @@ pub async fn process_transactions<'i, I, ProviderT, TransportT>( block: &Block, provider: &ProviderT, tx_traces: I, -) -> anyhow::Result>> +) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, @@ -146,7 +144,7 @@ pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, tx_trace: &BTreeMap, -) -> anyhow::Result> +) -> anyhow::Result> where ProviderT: Provider, TransportT: Transport + Clone, @@ -161,7 +159,7 @@ where .ok() .flatten(); - let jumpdest_table: Option = structlog_opt.and_then(|struct_log| { + let jc: Option<(JumpDestTableWitness, CodeDb)> = structlog_opt.and_then(|struct_log| { jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( |error| { info!( @@ -170,15 +168,18 @@ where ); None }, - |jdt| { + |(jdt, code_db)| { info!( "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); - Some(jdt) + Some((jdt, code_db)) }, ) }); - Ok(jumpdest_table) + // TODO + // let jumpdest_table = jc.map(|(j, c)| j); + + Ok(jc) } diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 238ac7b7a..ac7173cbf 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -20,6 +20,7 @@ use alloy_primitives::B256; use alloy_primitives::U256; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; +use evm_arithmetization::CodeDb; use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; @@ -109,10 +110,11 @@ pub(crate) fn generate_jumpdest_table( tx: &Transaction, struct_log: &[StructLog], tx_traces: &BTreeMap, -) -> anyhow::Result { +) -> anyhow::Result<(JumpDestTableWitness, CodeDb)> { trace!("Generating JUMPDEST table for tx: {}", tx.hash); let mut jumpdest_table = JumpDestTableWitness::default(); + let mut code_db = CodeDb::default(); // This map does not contain `initcodes`. let callee_addr_to_code_hash: HashMap = tx_traces @@ -136,10 +138,10 @@ pub(crate) fn generate_jumpdest_table( let entrypoint_code_hash: H256 = match tx.to { Some(to_address) if PRECOMPILE_ADDRESSES.contains(&to_address) => { - return Ok(jumpdest_table) + return Ok((jumpdest_table, code_db)) } Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { - return Ok(jumpdest_table) + return Ok((jumpdest_table, code_db)) } Some(to_address) => callee_addr_to_code_hash[&to_address], None => { @@ -297,6 +299,7 @@ pub(crate) fn generate_jumpdest_table( let memory: Vec = mem_res.unwrap().concat(); let init_code = &memory[offset..offset + size]; + code_db.insert(init_code.to_vec()); let init_code_hash = keccak(init_code); call_stack.push((init_code_hash, next_ctx_available)); @@ -374,7 +377,7 @@ pub(crate) fn generate_jumpdest_table( } } } - Ok(jumpdest_table) + Ok((jumpdest_table, code_db)) } /// This module exists as a workaround for parsing `StructLog`. The `error` diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 582b7ee8f..c8dab028c 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -76,11 +76,14 @@ where let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); - let (code_db, mut tx_traces) = match (pre_trace, diff_trace) { + let (mut code_db, mut tx_traces) = match (pre_trace, diff_trace) { ( GethTrace::PreStateTracer(PreStateFrame::Default(read)), GethTrace::PreStateTracer(PreStateFrame::Diff(diff)), - ) => process_tx_traces(access_list, read, diff).await?, + ) => { + info!("{:?} {:?} {:?}", tx.hash, read, diff); + process_tx_traces(access_list, read, diff).await? + } _ => unreachable!(), }; @@ -89,7 +92,7 @@ where tx_traces.insert(tx_receipt.contract_address.unwrap(), TxnTrace::default()); }; - let jumpdest_table: Option = structlog_opt.and_then(|struct_logs| { + let jc: Option<(JumpDestTableWitness, CodeDb)> = structlog_opt.and_then(|struct_logs| { jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces).map_or_else( |error| { info!( @@ -98,16 +101,21 @@ where ); None }, - |jdt| { + |(jdt, code_db)| { info!( "{:#?}: JumpDestTable generation succeeded with result: {}", tx.hash, jdt ); - Some(jdt) + Some((jdt, code_db)) }, ) }); + let jumpdest_table = jc.map(|(j, c)| { + code_db.extend(c); + j + }); + let tx_meta = TxnMeta { byte_code: ::TxEnvelope::try_from(tx.clone())?.encoded_2718(), new_receipt_trie_node_byte: alloy::rlp::encode(tx_receipt.inner), From 6bff4e4eac04fec5b881bd75bf9d7921011c0353 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 26 Sep 2024 17:24:44 +0200 Subject: [PATCH 035/112] fix --- scripts/prove_stdio.sh | 4 +-- scripts/test_jerigon.sh | 19 ++++++----- scripts/test_native.sh | 66 ++++++++++++++------------------------- trace_decoder/src/core.rs | 25 +++++++++------ 4 files changed, 53 insertions(+), 61 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 2b804f98a..25db16e19 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -35,9 +35,9 @@ TEST_OUT_PATH="${REPO_ROOT}/test.out" export RAYON_NUM_THREADS=$num_procs export TOKIO_WORKER_THREADS=$num_procs -export RUST_MIN_STACK=33554432 +#export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full -export RUST_LOG=info +#export RUST_LOG=trace # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 5426b5b01..e30300787 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -9,9 +9,11 @@ if [ -z $RPC ]; then fi mkdir -p witnesses -export RAYON_NUM_THREADS=4 +# Must match the values in prove_stdio.sh or build is dirty. +export RAYON_NUM_THREADS=1 export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full +#export RUST_LOG=info #export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' #export RUST_MIN_STACK=67108864 @@ -251,8 +253,8 @@ BLOCKS="$ROUND3" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" -printf "githash block verdict\n" | tee -a witnesses/jerigon_results.txt -echo "---------------------------" | tee -a witnesses/jerigon_results.txt +printf "githash block verdict duration\n" | tee -a witnesses/jerigon_results.txt +echo "------------------------------------" | tee -a witnesses/jerigon_results.txt for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` @@ -263,13 +265,16 @@ for BLOCK in $BLOCKS; do echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - ./prove_stdio.sh $WITNESS test_only + SECONDS=0 + timeout 10m ./prove_stdio.sh $WITNESS test_only EXITCODE=$? + DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + echo $DURATION if [ $EXITCODE -eq 0 ] then - RESULT="success" + VERDICT="success" else - RESULT="failure" + VERDICT="failure" fi - printf "%s %10i %s\n" $GITHASH $BLOCK $RESULT | tee -a witnesses/jerigon_results.txt + printf "%s %10i %s %s\n" $GITHASH $BLOCK $VERDICT $DURATION | tee -a witnesses/jerigon_results.txt done diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 56de59a49..63da02115 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -33,6 +33,11 @@ PRECANCUN=" 19240700 " +#It's visible with block 20727641 +ROUND1=`echo {20727640..20727650}` + + + CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` STATICTIP=20721266 @@ -42,57 +47,34 @@ RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." -BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" +#BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" +BLOCKS="$ROUND1" #BLOCKS="$CANCUNBLOCKS" echo "Testing blocks: $BLOCKS" -echo "Downloading witnesses.." -echo "------------------------"| tee -a witnesses/native_results.txt +echo "Testing: $BLOCKS" +printf "githash block verdict duration\n" | tee -a witnesses/native_results.txt +echo "------------------------------------" | tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do + GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" - until [ -f $WITNESS -a -s $WITNESS ]; do - echo "Fetching block $BLOCK" - cargo run --release --verbose --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - EXITCODE=$? - - if [ $EXITCODE -eq 0 -a -f $WITNESS -a -s $WITNESS ] - then - printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH success | tee -a witnesses/native_results.txt - break - else - printf "%10i %s witness saved: %s.\n" $BLOCK $GITHASH failure | tee -a witnesses/native_results.txt - fi - done - - echo "Witness for block $BLOCK ($WITNESS) prepared." - - echo "Testing $WITNESS" - ./prove_stdio.sh $WITNESS test_only + echo "Fetching block $BLOCK" + export RUST_LOG=rpc=trace + cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + echo "Testing blocks: $BLOCKS." + echo "Now testing block $BLOCK .." + export RUST_LOG=info + SECONDS=0 + timeout 30m ./prove_stdio.sh $WITNESS test_only EXITCODE=$? + DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + echo $DURATION if [ $EXITCODE -eq 0 ] then - RESULT="success" + VERDICT="success" else - RESULT="failure" + VERDICT="failure" fi - printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt + printf "%s %10i %s %s\n" $GITHASH $BLOCK $VERDICT $DURATION | tee -a witnesses/native_results.txt done - -#echo "Finished downloading witnesses." -#echo "Testing prepared witnesses.." -# -#for WITNESS in witnesses/*.native.$GITHASH.witness.json; do -# echo "Testing $WITNESS" -# ./prove_stdio.sh $WITNESS test_only -# EXITCODE=$? -# if [ $EXITCODE -eq 0 ] -# then -# RESULT="success" -# else -# RESULT="failure" -# fi -# printf "%10i %s witness tested: %s.\n" $BLOCK $GITHASH $RESULT | tee -a witnesses/native_results.txt -#done -# -#echo "Finished testing witnesses." diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index c6e530a75..6a32046d3 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -53,7 +53,7 @@ pub fn entrypoint( txn_info, } = trace; let (state, storage, mut code) = start(trie_pre_images)?; - code.extend(code_db); + code.extend(code_db.clone()); let OtherBlockData { b_data: @@ -132,24 +132,30 @@ pub fn entrypoint( } }); + // TODO convert to Hash2Code let initmap: HashMap<_, _> = initcodes .into_iter() .map(|it| (keccak_hash::keccak(&it), it)) .collect(); + log::trace!("Initmap {:?}", initmap); let contractmap: HashMap<_, _> = contract_code .into_iter() .map(|it| (keccak_hash::keccak(&it), it)) .collect(); - - log::trace!("Initmap {:?}", initmap); log::trace!("Contractmap {:?}", contractmap); - //log::trace!("DECODER: {:#?}", res); - let mut c = code.inner.clone(); - c.extend(contractmap); - c.extend(initmap); - c + let codemap: HashMap<_, _> = code_db + .clone() + .into_iter() + .map(|it| (keccak_hash::keccak(&it), it)) + .collect(); + log::trace!("Codemap {:?}", codemap); + + let mut res = codemap; + res.extend(contractmap); + res.extend(initmap); + res }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), @@ -656,10 +662,9 @@ fn map_receipt_bytes(bytes: Vec) -> anyhow::Result> { /// trace. /// If there are any txns that create contracts, then they will also /// get added here as we process the deltas. -#[derive(Clone)] struct Hash2Code { /// Key must always be [`hash`] of value. - pub inner: HashMap>, + inner: HashMap>, } impl Hash2Code { From c26f4752921db7c7f0e14255e1299490a52de4d7 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 26 Sep 2024 20:19:56 +0200 Subject: [PATCH 036/112] fix arguments --- evm_arithmetization/src/generation/prover_input.rs | 2 +- scripts/test_jerigon.sh | 2 +- zero/src/bin/leader/cli.rs | 2 +- zero/src/bin/rpc.rs | 2 +- zero/src/rpc/jerigon.rs | 7 ++++--- zero/src/rpc/mod.rs | 7 ++++--- zero/src/rpc/native/txn.rs | 6 +++--- 7 files changed, 15 insertions(+), 13 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 43511394a..9015ff4a3 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -812,7 +812,7 @@ impl GenerationState { if let (Some(rw), Some(sw)) = (rpcw, simw) { info!("SIMW {}", sw); info!("RPCW {}", rw); - assert_eq!(rw, &sw); + // assert_eq!(rw, &sw); } info!("SIM {:#?}", sim); diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index e30300787..685939540 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -261,7 +261,7 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace - cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index 78ad0fbdb..d89a349ea 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -44,7 +44,7 @@ pub(crate) enum Command { #[arg(long, short = 't', default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "prover-simulation"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] jumpdest_src: JumpdestSrc, /// The block interval for which to generate a proof. #[arg(long, short = 'i')] diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index a55073342..fe29247ee 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -38,7 +38,7 @@ struct RpcToolConfig { #[arg(short = 't', long, default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "simulation"), ("rpc_type", "native", "zero")], required = false)] + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "prover-simulation"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] jumpdest_src: JumpdestSrc, /// Backoff in milliseconds for retry requests. #[arg(long, default_value_t = 0)] diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 0b79cfd70..73280fc99 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -68,8 +68,8 @@ where .await?; let jdts: Vec> = match jumpdest_src { - JumpdestSrc::Simulation => vec![None; tx_results.len()], - JumpdestSrc::Zero => { + JumpdestSrc::ProverSimulation => vec![None; tx_results.len()], + JumpdestSrc::ClientFetchedStructlogs => { process_transactions( &block, cached_provider.get_provider().await?.deref(), @@ -77,7 +77,8 @@ where ) .await? } - JumpdestSrc::Jerigon => todo!("hybrid server bulk struct log retrieval/local jumpdest table generation not yet implemented"), + JumpdestSrc::ServerFetchedStructlogs => todo!("hybrid server bulk struct log retrieval/local jumpdest table generation not yet implemented"), + JumpdestSrc::Serverside => todo!(), }; let mut code_db = CodeDb::default(); diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index e55a1ac89..4f4908eb0 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -44,9 +44,10 @@ pub enum RpcType { /// The Jumpdest source type. #[derive(ValueEnum, Clone, Debug, Copy)] pub enum JumpdestSrc { - Simulation, - Zero, - Jerigon, + ProverSimulation, + ClientFetchedStructlogs, + ServerFetchedStructlogs, // later + Serverside, // later } /// Obtain the prover input for one block diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index c8dab028c..96d30f554 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -155,7 +155,7 @@ where let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); let (tx_receipt, pre_trace, diff_trace, structlog_trace) = match jumpdest_src { - JumpdestSrc::Zero => { + JumpdestSrc::ClientFetchedStructlogs => { let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); futures::try_join!( tx_receipt_fut, @@ -164,12 +164,12 @@ where structlog_trace_fut, )? } - JumpdestSrc::Simulation => { + JumpdestSrc::ProverSimulation => { let (tx_receipt, pre_trace, diff_trace) = futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; (tx_receipt, pre_trace, diff_trace, None) } - JumpdestSrc::Jerigon => todo!(), + _ => todo!(), }; Ok(( From f3674090ed2e1d482c027580b96bbaf1dd7c5dd5 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 30 Sep 2024 17:39:59 +0200 Subject: [PATCH 037/112] feedback --- trace_decoder/src/interface.rs | 10 ---------- zero/src/rpc/jumpdest.rs | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index e57ac7c81..1005b2afa 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -166,16 +166,6 @@ pub enum ContractCodeUsage { /// contract code will not appear in the [`BlockTrace`] map. Write(#[serde(with = "crate::hex")] Vec), } -// Review: Re-adding this. It has been removed upstream. -impl ContractCodeUsage { - /// Get code hash from a read or write operation of contract code. - pub fn get_code_hash(&self) -> H256 { - match self { - ContractCodeUsage::Read(hash) => *hash, - ContractCodeUsage::Write(bytes) => keccak(bytes), - } - } -} /// Other data that is needed for proof gen. #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index b9bf3c148..e4ef4c788 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -27,6 +27,7 @@ use keccak_hash::keccak; use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::is_precompile; +use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; use tracing::trace; @@ -53,6 +54,14 @@ fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDe } } +/// Get code hash from a read or write operation of contract code. +fn get_code_hash(usage: &ContractCodeUsage) -> H256 { + match usage { + ContractCodeUsage::Read(hash) => *hash, + ContractCodeUsage::Write(bytes) => keccak(bytes), + } +} + /// Predicate that determines whether a `StructLog` that includes memory is /// required. fn trace_contains_create(structlog: &[StructLog]) -> bool { @@ -124,7 +133,7 @@ pub(crate) fn generate_jumpdest_table( .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) .filter(|(_callee_addr, code_usage)| code_usage.is_some()) .map(|(callee_addr, code_usage)| { - (*callee_addr, code_usage.as_ref().unwrap().get_code_hash()) + (*callee_addr, get_code_hash(code_usage.as_ref().unwrap())) }) .collect(); @@ -204,6 +213,10 @@ pub(crate) fn generate_jumpdest_table( if evm_stack.len() < operands_used { trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len()); + // Note for future debugging: There may exist edge cases, where the call + // context has been incremented before the call op fails. This should be + // accounted for before this and the following `continue`. The details are + // defined in `sys_calls.asm`. continue; } // This is the same stack index (i.e. 2nd) for all four opcodes. See https://ethervm.io/#F1 @@ -217,6 +230,7 @@ pub(crate) fn generate_jumpdest_table( *address, U256::from(U160::MAX) ); + // Se note above. continue; }; let lower_20_bytes = U160::from(*address); @@ -368,6 +382,7 @@ pub(crate) fn generate_jumpdest_table( ); continue; } + assert!(jumpdest_offset.unwrap() < 24576); jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset.unwrap()); } "EXTCODECOPY" | "EXTCODESIZE" => { From 464dbc0d369ab448860a49c55a64676a492e1dcd Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 30 Sep 2024 19:18:00 +0200 Subject: [PATCH 038/112] fix --- scripts/prove_stdio.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 25db16e19..e94b5d5a4 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -37,7 +37,7 @@ export TOKIO_WORKER_THREADS=$num_procs #export RUST_MIN_STACK=33554432 export RUST_BACKTRACE=full -#export RUST_LOG=trace +export RUST_LOG=info # Script users are running locally, and might benefit from extra perf. # See also .cargo/config.toml. export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' From 2783ccdd1dcc0124cd09c89b538c7e4731db0b90 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 30 Sep 2024 23:57:10 +0200 Subject: [PATCH 039/112] debugging 460 --- .../src/generation/prover_input.rs | 24 +++++---- scripts/prove_stdio.sh | 17 +++--- scripts/test_jerigon.sh | 53 +++++++++++++++++-- scripts/test_native.sh | 4 +- trace_decoder/src/interface.rs | 2 +- zero/src/rpc/jumpdest.rs | 2 +- 6 files changed, 76 insertions(+), 26 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 9015ff4a3..5ae94e1b2 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -796,27 +796,29 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - dbg!(&self.inputs.jumpdest_table); - dbg!(&self.inputs.txn_hashes); + // dbg!(&self.inputs.jumpdest_table); + // dbg!(&self.inputs.txn_hashes); let rpcw = &self.inputs.jumpdest_table; let rpc: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); - info!("Generating JUMPDEST tables part2"); + info!("Generating JUMPDEST tables: Running SIM"); let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); - let (sim, simw): (Option, Option) = + let (sim, ref simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); - if let (Some(rw), Some(sw)) = (rpcw, simw) { - info!("SIMW {}", sw); - info!("RPCW {}", rw); - // assert_eq!(rw, &sw); - } + // if let (Some(rw), Some(sw)) = (rpcw, simw) { + // info!("SIMW {}", sw); + // info!("RPCW {}", rw); + // // assert_eq!(rw, &sw); + // } - info!("SIM {:#?}", sim); - info!("RPC {:#?}", rpc); + // info!("SIMW {:#?}", &simw); + // info!("RPCW {:#?}", rpcw); + assert_eq!(rpcw, simw); + assert_eq!(rpc, sim); self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index e94b5d5a4..077ce9261 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -21,15 +21,15 @@ fi REPO_ROOT=$(git rev-parse --show-toplevel) PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs" -BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}" +BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-1}" echo "Block batch size: $BLOCK_BATCH_SIZE" -BATCH_SIZE=${BATCH_SIZE:=1} +BATCH_SIZE=${BATCH_SIZE:-1} echo "Batch size: $BATCH_SIZE" OUTPUT_LOG="${REPO_ROOT}/output.log" PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" -TEST_OUT_PATH="${REPO_ROOT}/test.out" +TEST_OUT_PATH="${REPO_ROOT}/$3.test.out" # Configured Rayon and Tokio with rough defaults export RAYON_NUM_THREADS=$num_procs @@ -98,14 +98,17 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $TEST_OUT_PATH + cargo run --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR --batch-size $BATCH_SIZE --save-inputs-on-error stdio < $INPUT_FILE |& tee &> $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" - rm $TEST_OUT_PATH + #rm $TEST_OUT_PATH exit - else - echo "Failed to create proof witnesses. See \"zk_evm/test.out\" for more details." + elif grep -q 'Proving task finished with error' $TEST_OUT_PATH; then + echo -e "\n\nFailed to create proof witnesses. See \"zk_evm/test.out\" for more details." exit 1 + else + echo -e "\n\nUndecided. Proving process has stopped but verdict is undecided. See \"zk_evm/test.out\" for more details." + exit 2 fi fi diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 685939540..d1a6d272f 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -237,6 +237,51 @@ ROUND3=" 548 " +ROUND4=" +136 +186 +268 +282 +301 +304 +321 +333 +460 +461 +462 +463 +464 +465 +466 +467 +468 +473 +474 +528 +529 +530 +531 +532 +533 +534 +570 +664 +" + +ROUND5=" +460 +461 +462 +463 +464 +465 +466 +467 +468 +473 +474 +664 +" # 470..663 from Robin for i in {470..663} @@ -249,11 +294,11 @@ NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3" -BLOCKS="$ROUND3" +BLOCKS="$ROUND5" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" -printf "githash block verdict duration\n" | tee -a witnesses/jerigon_results.txt +printf "\ngithash block verdict duration\n" | tee -a witnesses/jerigon_results.txt echo "------------------------------------" | tee -a witnesses/jerigon_results.txt for BLOCK in $BLOCKS; do @@ -261,12 +306,12 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace + SECONDS=0 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info - SECONDS=0 - timeout 10m ./prove_stdio.sh $WITNESS test_only + timeout 10m ./prove_stdio.sh $WITNESS test_only $BLOCK EXITCODE=$? DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` echo $DURATION diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 63da02115..5e8bfeff8 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -10,8 +10,8 @@ fi mkdir -p witnesses # Must match the values in prove_stdio.sh or build is dirty. -export RAYON_NUM_THREADS=1 -export TOKIO_WORKER_THREADS=1 +#export RAYON_NUM_THREADS=1 +#export TOKIO_WORKER_THREADS=1 export RUST_BACKTRACE=full #export RUST_LOG=info #export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' diff --git a/trace_decoder/src/interface.rs b/trace_decoder/src/interface.rs index 1005b2afa..35248f9a3 100644 --- a/trace_decoder/src/interface.rs +++ b/trace_decoder/src/interface.rs @@ -8,7 +8,7 @@ use ethereum_types::{Address, U256}; use evm_arithmetization::jumpdest::JumpDestTableWitness; use evm_arithmetization::proof::{BlockHashes, BlockMetadata}; use evm_arithmetization::ConsolidatedHash; -use keccak_hash::{keccak, H256}; +use keccak_hash::H256; use mpt_trie::partial_trie::HashedPartialTrie; use serde::{Deserialize, Serialize}; diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index e4ef4c788..8e1c7670f 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -33,7 +33,7 @@ use tracing::trace; /// The maximum time we are willing to wait for a structlog before failing over /// to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(2 * 60); +const TIMEOUT_LIMIT: Duration = Duration::from_secs(10 * 60); /// Structure of Etheruem memory type Word = [u8; 32]; From abee81258d89d14551a674d107e9a8ca257c2f07 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 13:42:08 +0200 Subject: [PATCH 040/112] debugging 460 --- .../src/cpu/kernel/interpreter.rs | 6 ++- .../src/generation/prover_input.rs | 39 ++++++++++++------- trace_decoder/src/core.rs | 4 +- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 054c89c27..985ed2dba 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -98,7 +98,7 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let clock = interpreter.get_clock(); - let jdtw = interpreter + let (a, jdtw) = interpreter .generation_state .set_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); @@ -106,7 +106,9 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - (interpreter.generation_state.jumpdest_table).map(|x| (x, jdtw)) + // (interpreter.generation_state.jumpdest_table).map(|x| (x, jdtw)) + interpreter.generation_state.jumpdest_table = Some(a.clone()); + Some((a, jdtw)) } } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 5ae94e1b2..83213783c 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -359,9 +359,12 @@ impl GenerationState { } /// Returns the next used jump address. + /// todo fn run_next_jumpdest_table_address(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; + // get_code from self.memory + if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; } @@ -798,11 +801,14 @@ impl GenerationState { info!("Generating JUMPDEST tables"); // dbg!(&self.inputs.jumpdest_table); // dbg!(&self.inputs.txn_hashes); - let rpcw = &self.inputs.jumpdest_table; - let rpc: Option = rpcw - .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); - info!("Generating JUMPDEST tables: Running SIM"); + // let rpcw = &self.inputs.jumpdest_table.clone(); + // let rpc: Option = rpcw + // .as_ref() + // .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, + // &self.inputs.contract_code)); info!("Generating JUMPDEST tables: + // Running SIM"); + + self.inputs.jumpdest_table = None; let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); @@ -815,12 +821,15 @@ impl GenerationState { // // assert_eq!(rw, &sw); // } - // info!("SIMW {:#?}", &simw); + info!("SIMW {:#?}", &simw); // info!("RPCW {:#?}", rpcw); - assert_eq!(rpcw, simw); - assert_eq!(rpc, sim); + info!("SIMP {:#?}", &sim); + // info!("RPCP {:#?}", &rpc); + // assert_eq!(rpcw, simw); + // assert_eq!(rpc, sim); - self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + // self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + self.jumpdest_table = sim; Ok(()) } @@ -829,12 +838,12 @@ impl GenerationState { /// compute their respective proofs, by calling /// `get_proofs_and_jumpdests` pub(crate) fn set_jumpdest_analysis_inputs( - &mut self, + &self, jumpdest_table: HashMap>, - ) -> JumpDestTableWitness { + ) -> (JumpDestTableProcessed, JumpDestTableWitness) { let mut jdtw = JumpDestTableWitness::default(); - self.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from_iter( - jumpdest_table.into_iter().map(|(ctx, jumpdest_table)| { + let jdtp = JumpDestTableProcessed::new(HashMap::from_iter(jumpdest_table.into_iter().map( + |(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); let code_hash = keccak(code.clone()); trace!("ctx: {ctx}, code_hash: {:?} code: {:?}", code_hash, code); @@ -847,9 +856,9 @@ impl GenerationState { } else { (ctx, vec![]) } - }), + }, ))); - jdtw + (jdtp, jdtw) } pub(crate) fn get_current_code(&self) -> Result, ProgramError> { diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 0baac3f47..ded46a83b 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -65,7 +65,7 @@ pub fn entrypoint( txn_info, } = trace; let (state, storage, mut code) = start(trie_pre_images)?; - code.extend(code_db.clone()); + // code.extend(code_db.clone()); let OtherBlockData { b_data: @@ -837,7 +837,7 @@ impl Hash2Code { pub fn get(&mut self, hash: H256) -> anyhow::Result> { match self.inner.get(&hash) { Some(code) => Ok(code.clone()), - None => bail!("no code for hash {}", hash), + None => bail!("no code for hash {:#x}", hash), } } pub fn insert(&mut self, code: Vec) { From 8bfccddcac3405888318d52aae5c3654621a2509 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 13:44:51 +0200 Subject: [PATCH 041/112] dbg --- scripts/test_jerigon.sh | 2 +- scripts/test_native.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index d1a6d272f..ed7076e30 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -2,7 +2,7 @@ set -uo pipefail -export RPC= +RPC=${RPC_JERIGON} if [ -z $RPC ]; then # You must set an RPC endpoint exit 1 diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 5e8bfeff8..7f092f232 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -2,7 +2,7 @@ set -uo pipefail -export RPC= +RPC=${RPC_NATIVE} if [ -z $RPC ]; then # You must set an RPC endpoint exit 1 From f9c2f76f5bb2345c12e87d575bff4e27925368ee Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 14:57:32 +0200 Subject: [PATCH 042/112] bugfix --- trace_decoder/src/core.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index ded46a83b..fca50ac73 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -447,7 +447,7 @@ fn middle( storage_read, storage_written, code_usage, - self_destructed: _, + self_destructed, }, ) in traces .into_iter() @@ -544,6 +544,11 @@ fn middle( state_mask.insert(TrieKey::from_address(addr)); } } + + if self_destructed { + storage_tries.remove(&keccak_hash::keccak(addr)); + state_mask.extend(state_trie.reporting_remove(addr)?) + } } jumpdest_tables.push(jumpdest_table); From 313d78d3ed9febbc5a847250b7ee47150d062650 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 15:12:21 +0200 Subject: [PATCH 043/112] dbg --- .../src/generation/prover_input.rs | 23 +++++++++---------- scripts/test_jerigon.sh | 4 ++++ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 83213783c..fb8f69b7d 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -801,12 +801,11 @@ impl GenerationState { info!("Generating JUMPDEST tables"); // dbg!(&self.inputs.jumpdest_table); // dbg!(&self.inputs.txn_hashes); - // let rpcw = &self.inputs.jumpdest_table.clone(); - // let rpc: Option = rpcw - // .as_ref() - // .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, - // &self.inputs.contract_code)); info!("Generating JUMPDEST tables: - // Running SIM"); + let rpcw = &self.inputs.jumpdest_table.clone(); + let rpc: Option = rpcw + .as_ref() + .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; @@ -822,14 +821,14 @@ impl GenerationState { // } info!("SIMW {:#?}", &simw); - // info!("RPCW {:#?}", rpcw); + info!("RPCW {:#?}", rpcw); info!("SIMP {:#?}", &sim); - // info!("RPCP {:#?}", &rpc); - // assert_eq!(rpcw, simw); - // assert_eq!(rpc, sim); + info!("RPCP {:#?}", &rpc); + assert_eq!(rpcw, simw); + assert_eq!(rpc, sim); - // self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; - self.jumpdest_table = sim; + self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + // self.jumpdest_table = sim; Ok(()) } diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index ed7076e30..781ac92ce 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -283,6 +283,10 @@ ROUND5=" 664 " +ROUND6=" +664 +" + # 470..663 from Robin for i in {470..663} do From 09079e61ba4303b0593c60fbb97763774dc57e84 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 16:53:46 +0200 Subject: [PATCH 044/112] fix --- trace_decoder/src/core.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index fca50ac73..4b3303ddd 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -65,7 +65,7 @@ pub fn entrypoint( txn_info, } = trace; let (state, storage, mut code) = start(trie_pre_images)?; - // code.extend(code_db.clone()); + code.extend(code_db.clone()); let OtherBlockData { b_data: From 7c84a63a6d4318fa360c7c04d4b024c4d7bf83b3 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 17:58:11 +0200 Subject: [PATCH 045/112] batching working --- .../src/generation/prover_input.rs | 17 +++++++++-------- scripts/test_jerigon.sh | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index fb8f69b7d..425df5397 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -802,18 +802,19 @@ impl GenerationState { // dbg!(&self.inputs.jumpdest_table); // dbg!(&self.inputs.txn_hashes); let rpcw = &self.inputs.jumpdest_table.clone(); - let rpc: Option = rpcw + let rpcp: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; - let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); - let (sim, ref simw): (Option, Option) = + let (simp, ref simw): (Option, Option) = sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); + info!("Generating JUMPDEST tables: finished"); + // if let (Some(rw), Some(sw)) = (rpcw, simw) { // info!("SIMW {}", sw); // info!("RPCW {}", rw); @@ -822,12 +823,12 @@ impl GenerationState { info!("SIMW {:#?}", &simw); info!("RPCW {:#?}", rpcw); - info!("SIMP {:#?}", &sim); - info!("RPCP {:#?}", &rpc); - assert_eq!(rpcw, simw); - assert_eq!(rpc, sim); + info!("SIMP {:#?}", &simp); + info!("RPCP {:#?}", &rpcp); + // assert_eq!(simw, rpcw); + // assert_eq!(simp, rpcp); - self.jumpdest_table = if rpc.is_some() { rpc } else { sim }; + self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; // self.jumpdest_table = sim; Ok(()) diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 781ac92ce..8e02977c8 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -298,7 +298,7 @@ NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` #$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3" -BLOCKS="$ROUND5" +BLOCKS="$ROUND6" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" From 4202ece130bcb221adb1152aed29d9fc599d65f4 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 1 Oct 2024 22:04:26 +0200 Subject: [PATCH 046/112] cleanups --- .../src/generation/prover_input.rs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 425df5397..f3b5ec54a 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -799,9 +799,7 @@ impl GenerationState { // skipping the validate table call info!("Generating JUMPDEST tables"); - // dbg!(&self.inputs.jumpdest_table); - // dbg!(&self.inputs.txn_hashes); - let rpcw = &self.inputs.jumpdest_table.clone(); + let rpcw = self.inputs.jumpdest_table.clone(); let rpcp: Option = rpcw .as_ref() .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); @@ -815,21 +813,12 @@ impl GenerationState { info!("Generating JUMPDEST tables: finished"); - // if let (Some(rw), Some(sw)) = (rpcw, simw) { - // info!("SIMW {}", sw); - // info!("RPCW {}", rw); - // // assert_eq!(rw, &sw); - // } - info!("SIMW {:#?}", &simw); - info!("RPCW {:#?}", rpcw); + info!("RPCW {:#?}", &rpcw); info!("SIMP {:#?}", &simp); info!("RPCP {:#?}", &rpcp); - // assert_eq!(simw, rpcw); - // assert_eq!(simp, rpcp); self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; - // self.jumpdest_table = sim; Ok(()) } From e1124d3c631aae1123370c3a7c2878b079bfbc5c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 14:32:13 +0200 Subject: [PATCH 047/112] feedback docs --- .../src/generation/jumpdest.rs | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 9a3201cce..1966c265d 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -1,3 +1,27 @@ +//! EVM opcode 0x5B or 91 is [`JUMPDEST`] which encodes a a valid offset, that +//! opcodes `JUMP` and `JUMPI` can jump to. Jumps to non-[`JUMPDEST`] +//! instructions are invalid. During an execution a [`JUMPDEST`] may be visited +//! zero or more times. Offsets are measured in bytes with respect to the +//! beginning of some contract code, which is uniquely identified by its +//! `CodeHash`. Every time control flow is switches to another contract through +//! a `CALL`-like instruction a new call context, `Context`, is created. Thus, +//! the tripple (`CodeHash`,`Context`, `Offset`) uniquely identifies an visited +//! [`JUMPDEST`] offset of an execution. +//! +//! Since an operation like e.g. `PUSH 0x5B` does not encode a valid +//! [`JUMPDEST`] in its second byte, and `PUSH32 +//! 5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B` does not +//! encode valid [`JUMPDESTS`] in bytes 1-32, some deligence must be exercised +//! when proving validity of jump operations. +//! +//! This module concerns itself with data structures for collecting these +//! offsets for [`JUMPDEST`] that was visited during an execution and are not +//! recording duplicity. The proofs that each of these offsets are not rendered +//! invalid by any of the previous 32 bytes `PUSH1`-`PUSH32` is computed later +//! in [`prove_context_jumpdests`] on basis of these collections. +//! +//! [`JUMPDEST`]: https://www.evm.codes/?fork=cancun#5b + use std::cmp::max; use std::{ collections::{BTreeSet, HashMap}, @@ -9,12 +33,13 @@ use itertools::{sorted, Itertools}; use keccak_hash::H256; use serde::{Deserialize, Serialize}; -/// Each `CodeAddress` can be called one or more times, each time creating a new -/// `Context`. Each `Context` will contain one or more offsets of `JUMPDEST`. +/// Each `CodeAddress` can be called one or more times, +/// each time creating a new `Context`. +/// Each `Context` will contain one or more offsets of `JUMPDEST`. #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub struct ContextJumpDests(pub HashMap>); -/// The result after proving a `JumpDestTableWitness`. +/// The result after proving a [`JumpDestTableWitness`]. #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct JumpDestTableProcessed(HashMap>); From 1f8d476c869ff0567d5bb1d6ce52e7c8fb63be48 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 15:20:27 +0200 Subject: [PATCH 048/112] feedback --- evm_arithmetization/src/generation/jumpdest.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 1966c265d..a7a104a15 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -11,7 +11,7 @@ //! Since an operation like e.g. `PUSH 0x5B` does not encode a valid //! [`JUMPDEST`] in its second byte, and `PUSH32 //! 5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B` does not -//! encode valid [`JUMPDESTS`] in bytes 1-32, some deligence must be exercised +//! encode valid [`JUMPDESTS`] in bytes 1-32, some diligence must be exercised //! when proving validity of jump operations. //! //! This module concerns itself with data structures for collecting these @@ -64,10 +64,6 @@ impl JumpDestTableProcessed { } impl JumpDestTableWitness { - pub fn new(ctx_map: HashMap) -> Self { - Self(ctx_map) - } - pub fn get(&self, code_hash: &H256) -> Option<&ContextJumpDests> { self.0.get(code_hash) } From 7319a15a3eb99493822d7a72fae09d1d6e1acce1 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 16:06:59 +0200 Subject: [PATCH 049/112] feedback filtermap --- evm_arithmetization/src/generation/jumpdest.rs | 4 ++-- zero/src/rpc/jumpdest.rs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index a7a104a15..f980d51ac 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -33,7 +33,7 @@ use itertools::{sorted, Itertools}; use keccak_hash::H256; use serde::{Deserialize, Serialize}; -/// Each `CodeAddress` can be called one or more times, +/// Each `CodeHash` can be called one or more times, /// each time creating a new `Context`. /// Each `Context` will contain one or more offsets of `JUMPDEST`. #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] @@ -43,7 +43,7 @@ pub struct ContextJumpDests(pub HashMap>); #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct JumpDestTableProcessed(HashMap>); -/// Map `CodeAddress -> (Context -> [JumpDests])` +/// Map `CodeHash -> (Context -> [JumpDests])` #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub struct JumpDestTableWitness(HashMap); diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 8e1c7670f..e4b8e5e29 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -127,16 +127,19 @@ pub(crate) fn generate_jumpdest_table( let mut jumpdest_table = JumpDestTableWitness::default(); let mut code_db = CodeDb::default(); - // This map does not contain `initcodes`. + // This map does neither contain the `init` field of Contract Deployment + // transactions nor CREATE, CREATE2 payloads. let callee_addr_to_code_hash: HashMap = tx_traces .iter() - .map(|(callee_addr, trace)| (callee_addr, &trace.code_usage)) - .filter(|(_callee_addr, code_usage)| code_usage.is_some()) - .map(|(callee_addr, code_usage)| { - (*callee_addr, get_code_hash(code_usage.as_ref().unwrap())) + .filter_map(|(callee_addr, trace)| { + trace + .code_usage + .as_ref() + .map(|code| (*callee_addr, get_code_hash(&code))) }) .collect(); + // REVIEW: will be removed before merge trace!( "Transaction: {} is a {}.", tx.hash, @@ -166,9 +169,6 @@ pub(crate) fn generate_jumpdest_table( // true condition and target `jump_target`. let mut prev_jump: Option = None; - // Call depth of the previous `entry`. We initialize to 0 as this compares - // smaller to 1. - //let mut prev_depth = 0; // The next available context. Starts at 1. Never decrements. let mut next_ctx_available = 1; // Immediately use context 1; From d4838e0c83191afbe34cfd050b1efd1f3fdc49f3 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 2 Oct 2024 16:43:28 +0200 Subject: [PATCH 050/112] review --- zero/src/rpc/jumpdest.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index e4b8e5e29..3997e545c 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -191,6 +191,7 @@ pub(crate) fn generate_jumpdest_table( ); let (code_hash, ctx) = call_stack.last().unwrap(); + // REVIEW: will be removed before merge trace!("TX: {:?}", tx.hash); trace!("STEP: {:?}", step); trace!("STEPS: {:?}", struct_log.len()); From 27b5719457015015df0a1bf0be7cc51cde2d681f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 3 Oct 2024 12:26:57 +0200 Subject: [PATCH 051/112] fmt --- zero/src/rpc/jumpdest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 3997e545c..14b133705 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -135,7 +135,7 @@ pub(crate) fn generate_jumpdest_table( trace .code_usage .as_ref() - .map(|code| (*callee_addr, get_code_hash(&code))) + .map(|code| (*callee_addr, get_code_hash(code))) }) .collect(); From eaf3ed7175f3de8c196b709691b3eabc38654e89 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 3 Oct 2024 13:35:14 +0200 Subject: [PATCH 052/112] fix set_jumpdest_analysis_inputs_rpc --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 4 ++-- .../src/cpu/kernel/tests/core/jumpdest_analysis.rs | 3 ++- evm_arithmetization/src/generation/prover_input.rs | 8 +++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 985ed2dba..9ff445fa2 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -100,7 +100,7 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let (a, jdtw) = interpreter .generation_state - .set_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); + .get_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); log::debug!( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", @@ -163,7 +163,7 @@ pub(crate) fn set_registers_and_run( /// /// - `jumpdest_table_rpc`: The raw table received from RPC. /// - `code_db`: The corresponding database of contract code used in the trace. -pub(crate) fn set_jumpdest_analysis_inputs_rpc( +pub(crate) fn get_jumpdest_analysis_inputs_rpc( jumpdest_table_rpc: &JumpDestTableWitness, code_map: &HashMap>, ) -> JumpDestTableProcessed { diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index 607f8cc69..d1b076118 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -17,7 +17,8 @@ use crate::witness::operation::CONTEXT_SCALING_FACTOR; impl Interpreter { pub(crate) fn set_jumpdest_analysis_inputs(&mut self, jumps: HashMap>) { - let _ = self.generation_state.set_jumpdest_analysis_inputs(jumps); + let (jdtp, _jdtw) = self.generation_state.get_jumpdest_analysis_inputs(jumps); + self.generation_state.jumpdest_table = Some(jdtp); } pub(crate) fn get_jumpdest_bit(&self, offset: usize) -> U256 { diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index f3b5ec54a..e4206cb15 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -26,7 +26,7 @@ use crate::cpu::kernel::constants::cancun_constants::{ }; use crate::cpu::kernel::constants::context_metadata::ContextMetadata; use crate::cpu::kernel::interpreter::{ - set_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps, + get_jumpdest_analysis_inputs_rpc, simulate_cpu_and_get_user_jumps, }; use crate::curve_pairings::{bls381, CurveAff, CyclicGroup}; use crate::extension_tower::{FieldExt, Fp12, Fp2, BLS381, BLS_BASE, BLS_SCALAR, BN254, BN_BASE}; @@ -798,11 +798,13 @@ impl GenerationState { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call + // REVIEW: This will be rewritten to only run simulation when + // `self.inputs.jumpdest_table` is `None`. info!("Generating JUMPDEST tables"); let rpcw = self.inputs.jumpdest_table.clone(); let rpcp: Option = rpcw .as_ref() - .map(|jdt| set_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + .map(|jdt| get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); info!("Generating JUMPDEST tables: Running SIM"); self.inputs.jumpdest_table = None; @@ -826,7 +828,7 @@ impl GenerationState { /// Given a HashMap containing the contexts and the jumpdest addresses, /// compute their respective proofs, by calling /// `get_proofs_and_jumpdests` - pub(crate) fn set_jumpdest_analysis_inputs( + pub(crate) fn get_jumpdest_analysis_inputs( &self, jumpdest_table: HashMap>, ) -> (JumpDestTableProcessed, JumpDestTableWitness) { From 10b6a22cd4b9826ad7f8c77e02bc14872c15f112 Mon Sep 17 00:00:00 2001 From: 0xaatif <169152398+0xaatif@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:03:41 +0200 Subject: [PATCH 053/112] discuss: deser in #427 (#681) * refactor: StructPogPrime * keep module doc, avoid name `vec` --- zero/src/rpc/jumpdest.rs | 192 ++++++++++++++++----------------------- 1 file changed, 76 insertions(+), 116 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 14b133705..3f5057b27 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -13,6 +13,7 @@ use alloy::primitives::U160; use alloy::providers::ext::DebugApi; use alloy::providers::Provider; use alloy::rpc::types::eth::Transaction; +use alloy::rpc::types::trace::geth::GethTrace; use alloy::rpc::types::trace::geth::StructLog; use alloy::rpc::types::trace::geth::{GethDebugTracingOptions, GethDefaultTracingOptions}; use alloy::transports::RpcError; @@ -24,7 +25,6 @@ use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use evm_arithmetization::CodeDb; use keccak_hash::keccak; -use structlogprime::normalize_structlog; use tokio::time::timeout; use trace_decoder::is_precompile; use trace_decoder::ContractCodeUsage; @@ -86,7 +86,7 @@ where .await?; let stackonly_structlog_opt: Option> = - normalize_structlog(&stackonly_structlog_trace).await; + trace2structlog(stackonly_structlog_trace).unwrap_or_default(); let need_memory = stackonly_structlog_opt .as_deref() @@ -102,7 +102,7 @@ where structlog_tracing_options(true, need_memory, false), ); - let memory_structlog = normalize_structlog(&memory_structlog_fut.await?).await; + let memory_structlog = trace2structlog(memory_structlog_fut.await?).unwrap_or_default(); Ok::>, RpcError>(memory_structlog) }; @@ -398,131 +398,91 @@ pub(crate) fn generate_jumpdest_table( Ok((jumpdest_table, code_db)) } +fn trace2structlog(trace: GethTrace) -> Result>, serde_json::Error> { + match trace { + GethTrace::Default(it) => Ok(Some(it.struct_logs)), + GethTrace::JS(it) => Ok(Some(compat::deserialize(it)?.struct_logs)), + _ => Ok(None), + } +} /// This module exists as a workaround for parsing `StructLog`. The `error` -/// field is a string in Alloy but an object in Erigon. -pub mod structlogprime { - use core::option::Option::None; - use std::collections::BTreeMap; +/// field is a string in Geth and Alloy but an object in Erigon. A PR[^1] has +/// been merged to fix this upstream and should eventually render this +/// unnecessary. [^1]: `https://github.com/erigontech/erigon/pull/12089` +mod compat { + use std::{collections::BTreeMap, fmt, iter}; - use alloy::rpc::types::trace::geth::{DefaultFrame, GethTrace, StructLog}; + use alloy::rpc::types::trace::geth::{DefaultFrame, StructLog}; use alloy_primitives::{Bytes, B256, U256}; - use serde::{ser::SerializeMap as _, Deserialize, Serialize, Serializer}; - use serde_json::Value; - - /// Geth Default struct log trace frame - /// - /// - #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub(crate) struct DefaultFramePrime { - /// Whether the transaction failed - pub failed: bool, - /// How much gas was used. - pub gas: u64, - /// Output of the transaction - #[serde(serialize_with = "alloy_serde::serialize_hex_string_no_prefix")] - pub return_value: Bytes, - /// Recorded traces of the transaction - pub struct_logs: Vec, - } + use serde::{de::SeqAccess, Deserialize, Deserializer}; - /// Represents a struct log entry in a trace - /// - /// - #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] - pub(crate) struct StructLogPrime { - /// Program counter - pub pc: u64, - /// Opcode to be executed - pub op: String, - /// Remaining gas - pub gas: u64, - /// Cost for executing op - #[serde(rename = "gasCost")] - pub gas_cost: u64, - /// Current call depth - pub depth: u64, - /// Error message if any - #[serde(default, skip)] - pub error: Option, - /// EVM stack - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stack: Option>, - /// Last call's return data. Enabled via enableReturnData - #[serde( - default, - rename = "returnData", - skip_serializing_if = "Option::is_none" - )] - pub return_data: Option, - /// ref - #[serde(default, skip_serializing_if = "Option::is_none")] - pub memory: Option>, - /// Size of memory. - #[serde(default, rename = "memSize", skip_serializing_if = "Option::is_none")] - pub memory_size: Option, - /// Storage slots of current contract read from and written to. Only - /// emitted for SLOAD and SSTORE. Disabled via disableStorage - #[serde( - default, - skip_serializing_if = "Option::is_none", - serialize_with = "serialize_string_storage_map_opt" - )] - pub storage: Option>, - /// Refund counter - #[serde(default, rename = "refund", skip_serializing_if = "Option::is_none")] - pub refund_counter: Option, + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result { + _DefaultFrame::deserialize(d) } - /// Serializes a storage map as a list of key-value pairs _without_ - /// 0x-prefix - pub(crate) fn serialize_string_storage_map_opt( - storage: &Option>, - s: S, - ) -> Result { - match storage { - None => s.serialize_none(), - Some(storage) => { - let mut m = s.serialize_map(Some(storage.len()))?; - for (key, val) in storage.iter() { - let key = format!("{:?}", key); - let val = format!("{:?}", val); - // skip the 0x prefix - m.serialize_entry(&key.as_str()[2..], &val.as_str()[2..])?; - } - m.end() - } + /// The `error` field is a `string` in `geth` etc. but an `object` in + /// `erigon`. + fn error<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + #[derive(Deserialize)] + #[serde(untagged)] + enum Error { + String(String), + #[allow(dead_code)] + Object(serde_json::Map), } + Ok(match Error::deserialize(d)? { + Error::String(it) => Some(it), + Error::Object(_) => None, + }) } - impl TryInto for DefaultFramePrime { - fn try_into(self) -> Result { - let a = serde_json::to_string(&self)?; - let b: DefaultFramePrime = serde_json::from_str(&a)?; - let c = serde_json::to_string(&b)?; - let d: DefaultFrame = serde_json::from_str(&c)?; - Ok(d) - } - - type Error = anyhow::Error; + #[derive(Deserialize)] + #[serde(remote = "DefaultFrame", rename_all = "camelCase")] + struct _DefaultFrame { + failed: bool, + gas: u64, + return_value: Bytes, + #[serde(deserialize_with = "vec_structlog")] + struct_logs: Vec, } - pub fn try_reserialize(structlog_object: &Value) -> anyhow::Result { - let a = serde_json::to_string(structlog_object)?; - let b: DefaultFramePrime = serde_json::from_str(&a)?; - let d: DefaultFrame = b.try_into()?; - Ok(d) + fn vec_structlog<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = Vec; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("an array of `StructLog`") + } + fn visit_seq>(self, mut seq: A) -> Result { + #[derive(Deserialize)] + struct With(#[serde(with = "_StructLog")] StructLog); + let v = iter::from_fn(|| seq.next_element().transpose()) + .map(|it| it.map(|With(it)| it)) + .collect::>()?; + Ok(v) + } + } + + d.deserialize_seq(Visitor) } - pub(crate) async fn normalize_structlog( - unnormalized_structlog: &GethTrace, - ) -> Option> { - match unnormalized_structlog { - GethTrace::Default(structlog_frame) => Some(structlog_frame.struct_logs.clone()), - GethTrace::JS(structlog_js_object) => try_reserialize(structlog_js_object) - .ok() - .map(|s| s.struct_logs), - _ => None, - } + #[derive(Deserialize)] + #[serde(remote = "StructLog", rename_all = "camelCase")] + struct _StructLog { + pc: u64, + op: String, + gas: u64, + gas_cost: u64, + depth: u64, + #[serde(deserialize_with = "error")] + error: Option, + stack: Option>, + return_data: Option, + memory: Option>, + #[serde(rename = "memSize")] + memory_size: Option, + storage: Option>, + #[serde(rename = "refund")] + refund_counter: Option, } } From c11d17d80c7909a8559b2b5ccc4a85a1ad0d4ca7 Mon Sep 17 00:00:00 2001 From: Marko Atanasievski Date: Mon, 7 Oct 2024 20:31:25 +0200 Subject: [PATCH 054/112] feat: block structlog retrieval (#682) * feat: use block structlogs retrieval * fix: cleanup cli argumetns * fix: error handling * fix: error handling 2 * adjustments * avoid CREATE/2 * fixes * rename function * remove support for CREATE/2 --- zero/src/bin/leader/cli.rs | 2 +- zero/src/bin/rpc.rs | 2 +- zero/src/rpc/jerigon.rs | 129 ++++++++---------------- zero/src/rpc/jumpdest.rs | 198 ++++++++++--------------------------- zero/src/rpc/mod.rs | 3 +- zero/src/rpc/native/txn.rs | 169 ++++++++++++++++++------------- 6 files changed, 199 insertions(+), 304 deletions(-) diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index d89a349ea..cab645fbd 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -44,7 +44,7 @@ pub(crate) enum Command { #[arg(long, short = 't', default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "prover-simulation"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "client-fetched-structlogs"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] jumpdest_src: JumpdestSrc, /// The block interval for which to generate a proof. #[arg(long, short = 'i')] diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index fe29247ee..5dd548992 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -38,7 +38,7 @@ struct RpcToolConfig { #[arg(short = 't', long, default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "prover-simulation"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] + #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "client-fetched-structlogs"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] jumpdest_src: JumpdestSrc, /// Backoff in milliseconds for retry requests. #[arg(long, default_value_t = 0)] diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 73280fc99..5138346a9 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -1,30 +1,25 @@ use core::iter::Iterator; -use std::collections::BTreeMap; use std::ops::Deref as _; -use __compat_primitive_types::H160; +use alloy::eips::BlockNumberOrTag; use alloy::{ providers::Provider, - rpc::types::{eth::BlockId, trace::geth::StructLog, Block, BlockTransactionsKind, Transaction}, + rpc::types::{eth::BlockId, Block, BlockTransactionsKind}, transports::Transport, }; -use alloy_primitives::Address; use anyhow::Context as _; -use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; -use futures::stream::FuturesOrdered; -use futures::StreamExt as _; +use compat::Compat; +use evm_arithmetization::jumpdest::JumpDestTableWitness; use serde::Deserialize; use serde_json::json; -use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo, TxnTrace}; -use tracing::info; +use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo}; +use tracing::warn; -use super::{ - fetch_other_block_data, - jumpdest::{self, get_normalized_structlog}, - JumpdestSrc, -}; +use super::{fetch_other_block_data, JumpdestSrc}; use crate::prover::BlockProverInput; use crate::provider::CachedProvider; +use crate::rpc::jumpdest::{generate_jumpdest_table, get_block_normalized_structlogs}; + /// Transaction traces retrieved from Erigon zeroTracer. #[derive(Debug, Deserialize)] pub struct ZeroTxResult { @@ -67,30 +62,31 @@ where .get_block(target_block_id, BlockTransactionsKind::Full) .await?; - let jdts: Vec> = match jumpdest_src { + let block_jumpdest_table_witnesses: Vec> = match jumpdest_src { JumpdestSrc::ProverSimulation => vec![None; tx_results.len()], JumpdestSrc::ClientFetchedStructlogs => { + // In case of the error with retrieving structlogs from the server, + // continue without interruption. Equivalent to `ProverSimulation` case. process_transactions( &block, cached_provider.get_provider().await?.deref(), - tx_results.iter().map(|TxnInfo { traces, meta: _ }| traces), // &tx_traces, + &tx_results, ) - .await? + .await + .unwrap_or_else(|e| { + warn!("failed to fetch server structlogs for block {target_block_id}: {e}"); + Vec::new() + }) } - JumpdestSrc::ServerFetchedStructlogs => todo!("hybrid server bulk struct log retrieval/local jumpdest table generation not yet implemented"), JumpdestSrc::Serverside => todo!(), }; - let mut code_db = CodeDb::default(); // weave in the JDTs let txn_info = tx_results .into_iter() - .zip(jdts) - .map(|(mut tx_info, jdt)| { - tx_info.meta.jumpdest_table = jdt.map(|(j, c)| { - code_db.extend(c); - j - }); + .zip(block_jumpdest_table_witnesses) + .map(|(mut tx_info, jdtw)| { + tx_info.meta.jumpdest_table = jdtw; tx_info }) .collect(); @@ -106,81 +102,42 @@ where .context("invalid hex returned from call to eth_getWitness")?, }), txn_info, - code_db, + code_db: Default::default(), }, other_data, }) } -/// Processes the transactions in the given block and updates the code db. -pub async fn process_transactions<'i, I, ProviderT, TransportT>( +/// Processes the transactions in the given block, generating jumpdest tables +/// and updates the code database +pub async fn process_transactions<'i, ProviderT, TransportT>( block: &Block, provider: &ProviderT, - tx_traces: I, -) -> anyhow::Result>> + tx_results: &[TxnInfo], +) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, - I: Iterator>, { - let futures = block + let block_structlogs = + get_block_normalized_structlogs(provider, &BlockNumberOrTag::from(block.header.number)) + .await?; + + let tx_traces = tx_results + .iter() + .map(|tx| tx.traces.iter().map(|(h, t)| (h.compat(), t))); + + let block_jumpdest_tables = block .transactions .as_transactions() - .context("No transactions in block")? + .context("no transactions in block")? .iter() + .zip(block_structlogs) .zip(tx_traces) - .map(|(tx, tx_trace)| process_transaction(provider, tx, tx_trace)) - .collect::>(); - - futures - .collect::>() - .await - .into_iter() - .collect::, _>>() -} - -/// Processes the transaction with the given transaction hash and updates the -/// accounts state. -pub async fn process_transaction( - provider: &ProviderT, - tx: &Transaction, - tx_trace: &BTreeMap, -) -> anyhow::Result> -where - ProviderT: Provider, - TransportT: Transport + Clone, -{ - let tx_traces = tx_trace - .iter() - .map(|(h, t)| (Address::from(h.to_fixed_bytes()), t.clone())) - .collect(); - - let structlog_opt: Option> = get_normalized_structlog(provider, &tx.hash) - .await - .ok() - .flatten(); - - let jc: Option<(JumpDestTableWitness, CodeDb)> = structlog_opt.and_then(|struct_log| { - jumpdest::generate_jumpdest_table(tx, &struct_log, &tx_traces).map_or_else( - |error| { - info!( - "{:#?}: JumpDestTable generation failed with reason: {}", - tx.hash, error - ); - None - }, - |(jdt, code_db)| { - info!( - "{:#?}: JumpDestTable generation succeeded with result: {}", - tx.hash, jdt - ); - Some((jdt, code_db)) - }, - ) - }); - - // TODO - // let jumpdest_table = jc.map(|(j, c)| j); + .map(|((tx, structlog), tx_trace)| { + structlog.and_then(|it| generate_jumpdest_table(tx, &it.1, tx_trace).ok()) + }) + .collect::>(); - Ok(jc) + Ok(block_jumpdest_tables) } diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 3f5057b27..976125877 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -1,43 +1,31 @@ use core::default::Default; use core::option::Option::None; use core::str::FromStr as _; -use core::time::Duration; -use std::collections::BTreeMap; use std::collections::HashMap; use std::ops::Not as _; use __compat_primitive_types::H160; use __compat_primitive_types::H256; -use alloy::primitives::Address; -use alloy::primitives::U160; +use alloy::eips::BlockNumberOrTag; +use alloy::primitives::{Address, U160}; use alloy::providers::ext::DebugApi; use alloy::providers::Provider; use alloy::rpc::types::eth::Transaction; -use alloy::rpc::types::trace::geth::GethTrace; -use alloy::rpc::types::trace::geth::StructLog; -use alloy::rpc::types::trace::geth::{GethDebugTracingOptions, GethDefaultTracingOptions}; -use alloy::transports::RpcError; +use alloy::rpc::types::trace::geth::{ + GethDebugTracingOptions, GethDefaultTracingOptions, GethTrace, StructLog, TraceResult, +}; use alloy::transports::Transport; -use alloy::transports::TransportErrorKind; -use alloy_primitives::B256; -use alloy_primitives::U256; +use alloy_primitives::{TxHash, U256}; +use anyhow::bail; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; -use evm_arithmetization::CodeDb; use keccak_hash::keccak; -use tokio::time::timeout; use trace_decoder::is_precompile; use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; -use tracing::trace; - -/// The maximum time we are willing to wait for a structlog before failing over -/// to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(10 * 60); - -/// Structure of Etheruem memory -type Word = [u8; 32]; -const WORDSIZE: usize = std::mem::size_of::(); +use tracing::{trace, warn}; +#[derive(Debug, Clone)] +pub struct TxStructLogs(pub Option, pub Vec); /// Pass `true` for the components needed. fn structlog_tracing_options(stack: bool, memory: bool, storage: bool) -> GethDebugTracingOptions { @@ -62,80 +50,52 @@ fn get_code_hash(usage: &ContractCodeUsage) -> H256 { } } -/// Predicate that determines whether a `StructLog` that includes memory is -/// required. -fn trace_contains_create(structlog: &[StructLog]) -> bool { - structlog - .iter() - .any(|entry| entry.op == "CREATE" || entry.op == "CREATE2") -} - -/// Gets the lightest possible structlog for transcation `tx_hash`. -pub(crate) async fn get_normalized_structlog( +pub(crate) async fn get_block_normalized_structlogs( provider: &ProviderT, - tx_hash: &B256, -) -> Result>, RpcError> + block: &BlockNumberOrTag, +) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, { - let inner = async { - // Optimization: It may be a better default to pull the stack immediately. - let stackonly_structlog_trace = provider - .debug_trace_transaction(*tx_hash, structlog_tracing_options(true, false, false)) - .await?; - - let stackonly_structlog_opt: Option> = - trace2structlog(stackonly_structlog_trace).unwrap_or_default(); - - let need_memory = stackonly_structlog_opt - .as_deref() - .is_some_and(trace_contains_create); - trace!("Need structlog with memory: {need_memory}"); - - if need_memory.not() { - return Ok(stackonly_structlog_opt); - }; - - let memory_structlog_fut = provider.debug_trace_transaction( - *tx_hash, - structlog_tracing_options(true, need_memory, false), - ); - - let memory_structlog = trace2structlog(memory_structlog_fut.await?).unwrap_or_default(); - - Ok::>, RpcError>(memory_structlog) - }; + let block_stackonly_structlog_traces = provider + .debug_trace_block_by_number(*block, structlog_tracing_options(true, false, false)) + .await?; + + let block_normalized_stackonly_structlog_traces = block_stackonly_structlog_traces + .into_iter() + .map(|tx_trace_result| match tx_trace_result { + TraceResult::Success { + result, tx_hash, .. + } => Ok(trace_to_tx_structlog(tx_hash, result)), + TraceResult::Error { error, tx_hash } => Err(anyhow::anyhow!( + "error fetching structlog for tx: {tx_hash:?}. Error: {error:?}" + )), + }) + .collect::>, anyhow::Error>>()?; - match timeout(TIMEOUT_LIMIT, inner).await { - Err(ellapsed_error) => Err(RpcError::Transport(TransportErrorKind::Custom(Box::new( - ellapsed_error, - )))), - Ok(structlog_res) => Ok(structlog_res?), - } + Ok(block_normalized_stackonly_structlog_traces) } /// Generate at JUMPDEST table by simulating the call stack in EVM, /// using a Geth structlog as input. -pub(crate) fn generate_jumpdest_table( +pub(crate) fn generate_jumpdest_table<'a>( tx: &Transaction, struct_log: &[StructLog], - tx_traces: &BTreeMap, -) -> anyhow::Result<(JumpDestTableWitness, CodeDb)> { + tx_traces: impl Iterator, +) -> anyhow::Result { trace!("Generating JUMPDEST table for tx: {}", tx.hash); let mut jumpdest_table = JumpDestTableWitness::default(); - let mut code_db = CodeDb::default(); // This map does neither contain the `init` field of Contract Deployment // transactions nor CREATE, CREATE2 payloads. let callee_addr_to_code_hash: HashMap = tx_traces - .iter() .filter_map(|(callee_addr, trace)| { trace .code_usage .as_ref() - .map(|code| (*callee_addr, get_code_hash(code))) + .map(|code| (callee_addr, get_code_hash(code))) }) .collect(); @@ -152,10 +112,10 @@ pub(crate) fn generate_jumpdest_table( let entrypoint_code_hash: H256 = match tx.to { Some(to_address) if is_precompile(H160::from_str(&to_address.to_string())?) => { - return Ok((jumpdest_table, code_db)) + return Ok(jumpdest_table) } Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { - return Ok((jumpdest_table, code_db)) + return Ok(jumpdest_table) } Some(to_address) => callee_addr_to_code_hash[&to_address], None => { @@ -259,68 +219,10 @@ pub(crate) fn generate_jumpdest_table( next_ctx_available += 1; } "CREATE" | "CREATE2" => { - prev_jump = None; - ensure!(entry.stack.as_ref().is_some(), "No evm stack found."); - // We reverse the stack, so the order matches our assembly code. - let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); - let operands_used = 3; - - if evm_stack.len() < operands_used { - trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len() ); - continue; - }; - - let [_value, offset, size, ..] = evm_stack[..] else { - unreachable!() - }; - if *offset > U256::from(usize::MAX) { - trace!( - "{op}: Offset {offset} was too large to fit in usize {}.", - usize::MAX - ); - continue; - }; - let offset: usize = offset.to(); - - if *size > U256::from(usize::MAX) { - trace!( - "{op}: Size {size} was too large to fit in usize {}.", - usize::MAX - ); - continue; - }; - let size: usize = size.to(); - - let memory_size = entry.memory.as_ref().unwrap().len() * WORDSIZE; - - if entry.memory.is_none() || offset + size > memory_size { - trace!("Insufficient memory available for {op}. Contract has size {size} and is supposed to be stored between offset {offset} and {}, but memory size is only {memory_size}.", offset+size); - continue; - } - let memory_raw: &[String] = entry.memory.as_ref().unwrap(); - let memory_parsed: Vec> = memory_raw - .iter() - .map(|mem_line| { - let mem_line_parsed = U256::from_str_radix(mem_line, 16)?; - Ok(mem_line_parsed.to_be_bytes()) - }) - .collect(); - let mem_res: anyhow::Result> = memory_parsed.into_iter().collect(); - if mem_res.is_err() { - trace!( - "{op}: Parsing memory failed with error: {}", - mem_res.unwrap_err() - ); - continue; - } - let memory: Vec = mem_res.unwrap().concat(); - - let init_code = &memory[offset..offset + size]; - code_db.insert(init_code.to_vec()); - let init_code_hash = keccak(init_code); - call_stack.push((init_code_hash, next_ctx_available)); - - next_ctx_available += 1; + bail!(format!( + "{} requires memory, aborting JUMPDEST-table generation.", + tx.hash + )); } "JUMP" => { prev_jump = None; @@ -383,7 +285,7 @@ pub(crate) fn generate_jumpdest_table( ); continue; } - assert!(jumpdest_offset.unwrap() < 24576); + ensure!(jumpdest_offset.unwrap() < 24576); jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset.unwrap()); } "EXTCODECOPY" | "EXTCODESIZE" => { @@ -395,16 +297,24 @@ pub(crate) fn generate_jumpdest_table( } } } - Ok((jumpdest_table, code_db)) + Ok(jumpdest_table) } -fn trace2structlog(trace: GethTrace) -> Result>, serde_json::Error> { +fn trace_to_tx_structlog(tx_hash: Option, trace: GethTrace) -> Option { match trace { - GethTrace::Default(it) => Ok(Some(it.struct_logs)), - GethTrace::JS(it) => Ok(Some(compat::deserialize(it)?.struct_logs)), - _ => Ok(None), + GethTrace::Default(structlog_frame) => { + Some(TxStructLogs(tx_hash, structlog_frame.struct_logs)) + } + GethTrace::JS(it) => { + let default_frame = compat::deserialize(it) + .inspect_err(|e| warn!("failed to deserialize js default frame {e:?}")) + .ok()?; + Some(TxStructLogs(tx_hash, default_frame.struct_logs)) + } + _ => None, } } + /// This module exists as a workaround for parsing `StructLog`. The `error` /// field is a string in Geth and Alloy but an object in Erigon. A PR[^1] has /// been merged to fix this upstream and should eventually render this @@ -474,7 +384,7 @@ mod compat { gas: u64, gas_cost: u64, depth: u64, - #[serde(deserialize_with = "error")] + #[serde(default, deserialize_with = "error")] error: Option, stack: Option>, return_data: Option, diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index 4f4908eb0..4e8a0278e 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -46,8 +46,7 @@ pub enum RpcType { pub enum JumpdestSrc { ProverSimulation, ClientFetchedStructlogs, - ServerFetchedStructlogs, // later - Serverside, // later + Serverside, // later } /// Obtain the prover input for one block diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 96d30f554..ffd692892 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -2,6 +2,8 @@ use core::option::Option::None; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use __compat_primitive_types::{H256, U256}; +use alloy::eips::BlockNumberOrTag; +use alloy::rpc::types::trace::geth::TraceResult; use alloy::{ primitives::{keccak256, Address, B256}, providers::{ @@ -19,17 +21,41 @@ use alloy::{ }, transports::Transport, }; -use anyhow::{Context as _, Ok}; +use anyhow::{bail, Context as _, Ok}; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; -use tracing::info; +use tracing::{error, warn}; +use crate::rpc::jumpdest::get_block_normalized_structlogs; use crate::rpc::Compat; use crate::rpc::{ - jumpdest::{self, get_normalized_structlog}, + jumpdest::{self}, JumpdestSrc, }; +pub(crate) async fn get_block_prestate_traces( + provider: &ProviderT, + block: &BlockNumberOrTag, + tracing_options: GethDebugTracingOptions, +) -> anyhow::Result> +where + ProviderT: Provider, + TransportT: Transport + Clone, +{ + let block_prestate_traces = provider + .debug_trace_block_by_number(*block, tracing_options) + .await?; + + block_prestate_traces + .into_iter() + .map(|trace_result| match trace_result { + TraceResult::Success { result, .. } => Ok(result), + TraceResult::Error { error, .. } => { + bail!("error fetching block prestate traces: {:?}", error) + } + }) + .collect::, anyhow::Error>>() +} /// Processes the transactions in the given block and updates the code db. pub async fn process_transactions( @@ -41,12 +67,58 @@ where ProviderT: Provider, TransportT: Transport + Clone, { + // Get block prestate traces + let block_prestate_trace = get_block_prestate_traces( + provider, + &BlockNumberOrTag::from(block.header.number), + prestate_tracing_options(false), + ) + .await?; + + // Get block diff traces + let block_diff_trace = get_block_prestate_traces( + provider, + &BlockNumberOrTag::from(block.header.number), + prestate_tracing_options(true), + ) + .await?; + + let block_structlogs = match jumpdest_src { + JumpdestSrc::ProverSimulation => vec![None; block_prestate_trace.len()], + JumpdestSrc::ClientFetchedStructlogs => { + // In case of the error with retrieving structlogs from the server, + // continue without interruption. Equivalent to `ProverSimulation` case. + get_block_normalized_structlogs(provider, &BlockNumberOrTag::from(block.header.number)) + .await + .unwrap_or_else(|e| { + warn!( + "failed to fetch server structlogs for block {}: {e}", + block.header.number + ); + vec![None; block_prestate_trace.len()] + }) + .into_iter() + .map(|tx_struct_log| tx_struct_log.map(|it| it.1)) + .collect() + } + JumpdestSrc::Serverside => todo!(), + }; + block .transactions .as_transactions() .context("No transactions in block")? .iter() - .map(|tx| process_transaction(provider, tx, jumpdest_src)) + .zip( + block_prestate_trace.into_iter().zip( + block_diff_trace + .into_iter() + .zip(block_structlogs.into_iter()), + ), + ) + .map(|(tx, (pre_trace, (diff_trace, structlog)))| { + process_transaction(provider, tx, pre_trace, diff_trace, structlog) + }) .collect::>() .try_fold( (BTreeSet::new(), Vec::new()), @@ -64,26 +136,24 @@ where pub async fn process_transaction( provider: &ProviderT, tx: &Transaction, - jumpdest_src: JumpdestSrc, + pre_trace: GethTrace, + diff_trace: GethTrace, + structlog_opt: Option>, ) -> anyhow::Result<(CodeDb, TxnInfo)> where ProviderT: Provider, TransportT: Transport + Clone, { - let (tx_receipt, pre_trace, diff_trace, structlog_opt) = - fetch_tx_data(provider, &tx.hash, jumpdest_src).await?; + let tx_receipt = fetch_tx_receipt(provider, &tx.hash).await?; let tx_status = tx_receipt.status(); let tx_receipt = tx_receipt.map_inner(rlp::map_receipt_envelope); let access_list = parse_access_list(tx.access_list.as_ref()); - let (mut code_db, mut tx_traces) = match (pre_trace, diff_trace) { + let (code_db, mut tx_traces) = match (pre_trace, diff_trace) { ( GethTrace::PreStateTracer(PreStateFrame::Default(read)), GethTrace::PreStateTracer(PreStateFrame::Diff(diff)), - ) => { - info!("{:?} {:?} {:?}", tx.hash, read, diff); - process_tx_traces(access_list, read, diff).await? - } + ) => process_tx_traces(access_list, read, diff).await?, _ => unreachable!(), }; @@ -92,28 +162,19 @@ where tx_traces.insert(tx_receipt.contract_address.unwrap(), TxnTrace::default()); }; - let jc: Option<(JumpDestTableWitness, CodeDb)> = structlog_opt.and_then(|struct_logs| { - jumpdest::generate_jumpdest_table(tx, &struct_logs, &tx_traces).map_or_else( - |error| { - info!( - "{:#?}: JumpDestTable generation failed with reason: {}", - tx.hash, error - ); - None - }, - |(jdt, code_db)| { - info!( - "{:#?}: JumpDestTable generation succeeded with result: {}", - tx.hash, jdt - ); - Some((jdt, code_db)) - }, - ) - }); - - let jumpdest_table = jc.map(|(j, c)| { - code_db.extend(c); - j + let jumpdest_table: Option = structlog_opt.and_then(|struct_logs| { + jumpdest::generate_jumpdest_table(tx, &struct_logs, tx_traces.iter().map(|(a, t)| (*a, t))) + .map_or_else( + |error| { + error!( + "{}: JumpDestTable generation failed with reason: {:?}", + tx.hash.to_string(), + error + ); + None + }, + Some, + ) }); let tx_meta = TxnMeta { @@ -136,48 +197,16 @@ where } /// Fetches the transaction data for the given transaction hash. -async fn fetch_tx_data( +async fn fetch_tx_receipt( provider: &ProviderT, tx_hash: &B256, - jumpdest_src: JumpdestSrc, -) -> anyhow::Result<( - ::ReceiptResponse, - GethTrace, - GethTrace, - Option>, -)> +) -> anyhow::Result<::ReceiptResponse> where ProviderT: Provider, TransportT: Transport + Clone, { - let tx_receipt_fut = provider.get_transaction_receipt(*tx_hash); - let pre_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(false)); - let diff_trace_fut = provider.debug_trace_transaction(*tx_hash, prestate_tracing_options(true)); - - let (tx_receipt, pre_trace, diff_trace, structlog_trace) = match jumpdest_src { - JumpdestSrc::ClientFetchedStructlogs => { - let structlog_trace_fut = get_normalized_structlog(provider, tx_hash); - futures::try_join!( - tx_receipt_fut, - pre_trace_fut, - diff_trace_fut, - structlog_trace_fut, - )? - } - JumpdestSrc::ProverSimulation => { - let (tx_receipt, pre_trace, diff_trace) = - futures::try_join!(tx_receipt_fut, pre_trace_fut, diff_trace_fut,)?; - (tx_receipt, pre_trace, diff_trace, None) - } - _ => todo!(), - }; - - Ok(( - tx_receipt.context("Transaction receipt not found.")?, - pre_trace, - diff_trace, - structlog_trace, - )) + let tx_receipt = provider.get_transaction_receipt(*tx_hash).await?; + Ok(tx_receipt.context("Transaction receipt not found.")?) } /// Parse the access list data into a hashmap. From 06b19134b18f7e27f3d96b555473172868eec769 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 02:12:51 +0200 Subject: [PATCH 055/112] better tracing --- Cargo.lock | 13 +++++++++++++ Cargo.toml | 2 +- zero/src/rpc/jerigon.rs | 16 ++++++++++++++-- zero/src/rpc/jumpdest.rs | 27 +++++++++++++++++++-------- zero/src/rpc/native/txn.rs | 4 ++-- 5 files changed, 49 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d72342620..ffc338cd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5160,6 +5160,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.18" @@ -5170,12 +5180,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7eca11df4..168811499 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,7 +98,7 @@ tokio = { version = "1.38.0", features = ["full"] } toml = "0.8.14" tower = "0.4" tracing = { version = "0.1", features = ["attributes"] } -tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } u4 = "0.1.0" uint = "0.9.5" url = "2.5.2" diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 5138346a9..0bbdefe60 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -13,7 +13,7 @@ use evm_arithmetization::jumpdest::JumpDestTableWitness; use serde::Deserialize; use serde_json::json; use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo}; -use tracing::warn; +use tracing::{debug, warn}; use super::{fetch_other_block_data, JumpdestSrc}; use crate::prover::BlockProverInput; @@ -135,7 +135,19 @@ where .zip(block_structlogs) .zip(tx_traces) .map(|((tx, structlog), tx_trace)| { - structlog.and_then(|it| generate_jumpdest_table(tx, &it.1, tx_trace).ok()) + structlog.and_then(|it| { + generate_jumpdest_table(tx, &it.1, tx_trace).map_or_else( + |error| { + debug!( + "{}: JumpDestTable generation failed with reason: {:?}", + tx.hash.to_string(), + error + ); + None + }, + Some, + ) + }) }) .collect::>(); diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 976125877..6188d7ae8 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -152,14 +152,25 @@ pub(crate) fn generate_jumpdest_table<'a>( let (code_hash, ctx) = call_stack.last().unwrap(); // REVIEW: will be removed before merge - trace!("TX: {:?}", tx.hash); - trace!("STEP: {:?}", step); - trace!("STEPS: {:?}", struct_log.len()); - trace!("OPCODE: {}", entry.op.as_str()); - trace!("CODE: {:?}", code_hash); - trace!("CTX: {:?}", ctx); - trace!("CURR_DEPTH: {:?}", curr_depth); - trace!("{:#?}\n", entry); + trace!( + step, + curr_depth, + tx_hash = ?tx.hash, + ?code_hash, + ctx, + entry.pc, + op, + // + ?entry, + ); + // trace!("TX: {:?}", tx.hash); + // trace!("STEP: {:?}", step); + // trace!("STEPS: {:?}", struct_log.len()); + // trace!("OPCODE: {}", entry.op.as_str()); + // trace!("CODE: {:?}", code_hash); + // trace!("CTX: {:?}", ctx); + // trace!("CURR_DEPTH: {:?}", curr_depth); + // trace!("{:#?}\n", entry); match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index ffd692892..bf69ec18e 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -25,7 +25,7 @@ use anyhow::{bail, Context as _, Ok}; use evm_arithmetization::{jumpdest::JumpDestTableWitness, CodeDb}; use futures::stream::{FuturesOrdered, TryStreamExt}; use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; -use tracing::{error, warn}; +use tracing::{debug, warn}; use crate::rpc::jumpdest::get_block_normalized_structlogs; use crate::rpc::Compat; @@ -166,7 +166,7 @@ where jumpdest::generate_jumpdest_table(tx, &struct_logs, tx_traces.iter().map(|(a, t)| (*a, t))) .map_or_else( |error| { - error!( + debug!( "{}: JumpDestTable generation failed with reason: {:?}", tx.hash.to_string(), error From 339f6af55b3ffeb85f899a1d45544cc8072fce98 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 02:14:05 +0200 Subject: [PATCH 056/112] bug fix --- .../src/generation/prover_input.rs | 20 ++++++++++--- scripts/test_jerigon.sh | 4 +-- scripts/test_native.sh | 26 +++++++++++++---- zero/src/rpc/jerigon.rs | 2 +- zero/src/rpc/jumpdest.rs | 28 +++++++++++-------- 5 files changed, 56 insertions(+), 24 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index e4206cb15..296de9959 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -815,10 +815,22 @@ impl GenerationState { info!("Generating JUMPDEST tables: finished"); - info!("SIMW {:#?}", &simw); - info!("RPCW {:#?}", &rpcw); - info!("SIMP {:#?}", &simp); - info!("RPCP {:#?}", &rpcp); + if simw != &rpcw { + if let Some(s) = simw { + info!("SIMW {}", s); + } + if let Some(r) = rpcw.as_ref() { + info!("RPCW {}", r); + } + info!("SIMW == RPCW ? {}", simw == &rpcw); + info!("tx: {:?}", self.inputs.txn_hashes); + panic!(); + // info!("SIMP {:?}", &simp); + // info!("RPCP {:?}", &rpcp); + // info!("SIMP == RPCP ? {}", &simp == &rpcp); + } else { + info!("JUMPDEST tables similar"); + } self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh index 8e02977c8..8a515f05e 100755 --- a/scripts/test_jerigon.sh +++ b/scripts/test_jerigon.sh @@ -297,8 +297,8 @@ TIP=688 NUMRANDOMBLOCKS=10 RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` -#$CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3" -BLOCKS="$ROUND6" +#BLOCKS="72 185" #$ROUND5 $CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3 $ROUND5 $ROUND4" +BLOCKS="$CIBLOCKS" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing: $BLOCKS" diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 7f092f232..eb65b5758 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash -set -uo pipefail +set -uxo pipefail -RPC=${RPC_NATIVE} if [ -z $RPC ]; then # You must set an RPC endpoint exit 1 @@ -33,9 +32,23 @@ PRECANCUN=" 19240700 " + #It's visible with block 20727641 ROUND1=`echo {20727640..20727650}` +ROUND2=" +20727641 +20727643 +20727644 +20727645 +20727646 +20727647 +20727648 +20727649 +20727650 +" + + CANCUN=19426587 @@ -48,12 +61,12 @@ GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" -BLOCKS="$ROUND1" +BLOCKS="20727641" #BLOCKS="$CANCUNBLOCKS" echo "Testing blocks: $BLOCKS" echo "Testing: $BLOCKS" -printf "githash block verdict duration\n" | tee -a witnesses/native_results.txt +printf "\n\ngithash block verdict duration\n" | tee -a witnesses/native_results.txt echo "------------------------------------" | tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do @@ -61,12 +74,13 @@ for BLOCK in $BLOCKS; do WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace - cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + rg "jump" $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info SECONDS=0 - timeout 30m ./prove_stdio.sh $WITNESS test_only + timeout 10m ./prove_stdio.sh $WITNESS test_only $BLOCK EXITCODE=$? DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` echo $DURATION diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 0bbdefe60..21480ab4a 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -75,7 +75,7 @@ where .await .unwrap_or_else(|e| { warn!("failed to fetch server structlogs for block {target_block_id}: {e}"); - Vec::new() + vec![None; tx_results.len()] }) } JumpdestSrc::Serverside => todo!(), diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 6188d7ae8..1462d120f 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -4,10 +4,11 @@ use core::str::FromStr as _; use std::collections::HashMap; use std::ops::Not as _; +use ::compat::Compat; use __compat_primitive_types::H160; use __compat_primitive_types::H256; use alloy::eips::BlockNumberOrTag; -use alloy::primitives::{Address, U160}; +use alloy::primitives::Address; use alloy::providers::ext::DebugApi; use alloy::providers::Provider; use alloy::rpc::types::eth::Transaction; @@ -196,16 +197,21 @@ pub(crate) fn generate_jumpdest_table<'a>( unreachable!() }; - if *address > U256::from(U160::MAX) { - trace!( - "{op}: Callee address {} was larger than possible {}.", - *address, - U256::from(U160::MAX) - ); - // Se note above. - continue; - }; - let lower_20_bytes = U160::from(*address); + // if *address > U256::from(U160::MAX) { + // trace!( + // "{op}: Callee address {} was larger than possible {}.", + // *address, + // U256::from(U160::MAX) + // ); + // // Se note above. + // continue; + // }; + let a: [u8; 32] = address.compat().into(); + // a <<= 96; + // a >>= 96; + // let aa: [u8; 32] = a.into(); + let aaa: H256 = a.into(); + let lower_20_bytes: [u8; 20] = H160::from(aaa).into(); let callee_address = Address::from(lower_20_bytes); if callee_addr_to_code_hash.contains_key(&callee_address) { From 61a6b6a32393d03744dd5fc66f2a63d1e50f7b4b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 02:14:58 +0200 Subject: [PATCH 057/112] json --- zero/src/bin/rpc.rs | 2 +- zero/src/tracing.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index 5dd548992..0b49a3a0a 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -219,8 +219,8 @@ async fn main() -> anyhow::Result<()> { // With the default configuration trace information is written // to stdout, but we already use stdout to write our payload (the witness). .with_writer(std::io::stderr) + .json() .with_ansi(false) - .compact() .with_filter(EnvFilter::from_default_env()), ) .init(); diff --git a/zero/src/tracing.rs b/zero/src/tracing.rs index 6c3f9a8bc..aa404059d 100644 --- a/zero/src/tracing.rs +++ b/zero/src/tracing.rs @@ -5,7 +5,7 @@ pub fn init() { .with( tracing_subscriber::fmt::layer() .with_ansi(false) - .compact() + .json() .with_filter(EnvFilter::from_default_env()), ) .init(); From f8f0a85cbb5096d4685013823872f6f76136ce5a Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 17:13:01 +0200 Subject: [PATCH 058/112] reinstantiate timeout --- zero/src/rpc/jumpdest.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 1462d120f..549506177 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -3,6 +3,7 @@ use core::option::Option::None; use core::str::FromStr as _; use std::collections::HashMap; use std::ops::Not as _; +use std::time::Duration; use ::compat::Compat; use __compat_primitive_types::H160; @@ -21,10 +22,15 @@ use anyhow::bail; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use keccak_hash::keccak; +use tokio::time::timeout; use trace_decoder::is_precompile; use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; use tracing::{trace, warn}; + +/// The maximum time we are willing to wait for a block's structlogs before +/// failing over to simulating the JumpDest analysis. +const TIMEOUT_LIMIT: Duration = Duration::from_secs(60); #[derive(Debug, Clone)] pub struct TxStructLogs(pub Option, pub Vec); @@ -59,9 +65,17 @@ where ProviderT: Provider, TransportT: Transport + Clone, { - let block_stackonly_structlog_traces = provider - .debug_trace_block_by_number(*block, structlog_tracing_options(true, false, false)) - .await?; + let block_stackonly_structlog_traces_fut = + provider.debug_trace_block_by_number(*block, structlog_tracing_options(true, false, false)); + + let block_stackonly_structlog_traces = + match timeout(TIMEOUT_LIMIT, block_stackonly_structlog_traces_fut).await { + Ok(traces) => traces?, + Err(elapsed) => { + trace!(target: "fetching block structlogs timed out", ?elapsed); + bail!(elapsed); + } + }; let block_normalized_stackonly_structlog_traces = block_stackonly_structlog_traces .into_iter() From dbb65ea5156ae94344aa3671657fc6a3ebfcb9aa Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 18:01:30 +0200 Subject: [PATCH 059/112] ignore None --- evm_arithmetization/src/generation/prover_input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 296de9959..6e2e32c47 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -815,7 +815,7 @@ impl GenerationState { info!("Generating JUMPDEST tables: finished"); - if simw != &rpcw { + if rpcw.is_some() && simw != &rpcw { if let Some(s) = simw { info!("SIMW {}", s); } From d415d2231886f2c502e36654c512c1390c7747e8 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 23:23:49 +0200 Subject: [PATCH 060/112] feedback --- .../src/cpu/kernel/tests/core/jumpdest_analysis.rs | 10 +++++----- evm_arithmetization/src/generation/prover_input.rs | 4 ++-- evm_arithmetization/tests/global_exit_root.rs | 2 +- zero/src/bin/leader/cli.rs | 7 ++++++- zero/src/bin/rpc.rs | 8 ++++++-- zero/src/rpc/native/txn.rs | 2 +- zero/src/tracing.rs | 1 - 7 files changed, 21 insertions(+), 13 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index d1b076118..0bb07eaf5 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -128,14 +128,14 @@ fn test_jumpdest_analysis() -> Result<()> { // We need to manually pop the jumpdest_table and push its value on the top of // the stack - (*interpreter + interpreter .generation_state .jumpdest_table .as_mut() - .unwrap()) - .get_mut(&CONTEXT) - .unwrap() - .pop(); + .unwrap() + .get_mut(&CONTEXT) + .unwrap() + .pop(); interpreter .push(41.into()) .expect("The stack should not overflow"); diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 6e2e32c47..1e4090aa8 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -375,12 +375,12 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = (*jumpdest_table).get_mut(&context) + if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) && let Some(next_jumpdest_address) = ctx_jumpdest_table.pop() { Ok((next_jumpdest_address + 1).into()) } else { - (*jumpdest_table).remove(&context); + jumpdest_table.remove(&context); Ok(U256::zero()) } } diff --git a/evm_arithmetization/tests/global_exit_root.rs b/evm_arithmetization/tests/global_exit_root.rs index c1f656efe..481643517 100644 --- a/evm_arithmetization/tests/global_exit_root.rs +++ b/evm_arithmetization/tests/global_exit_root.rs @@ -113,7 +113,7 @@ fn test_global_exit_root() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - batch_jumpdest_table: None, + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index cab645fbd..dac8c0b41 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -44,7 +44,12 @@ pub(crate) enum Command { #[arg(long, short = 't', default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "client-fetched-structlogs"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] + #[arg( + short = 'j', + long, + default_value = "client-fetched-structlogs", + required = false + )] jumpdest_src: JumpdestSrc, /// The block interval for which to generate a proof. #[arg(long, short = 'i')] diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index 0b49a3a0a..49c6fa995 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -38,7 +38,12 @@ struct RpcToolConfig { #[arg(short = 't', long, default_value = "jerigon")] rpc_type: RpcType, /// The source of jumpdest tables. - #[arg(short = 'j', long, default_value_ifs = [("rpc_type", "jerigon", "client-fetched-structlogs"), ("rpc_type", "native", "client-fetched-structlogs")], required = false)] + #[arg( + short = 'j', + long, + default_value = "client-fetched-structlogs", + required = false + )] jumpdest_src: JumpdestSrc, /// Backoff in milliseconds for retry requests. #[arg(long, default_value_t = 0)] @@ -219,7 +224,6 @@ async fn main() -> anyhow::Result<()> { // With the default configuration trace information is written // to stdout, but we already use stdout to write our payload (the witness). .with_writer(std::io::stderr) - .json() .with_ansi(false) .with_filter(EnvFilter::from_default_env()), ) diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index bf69ec18e..a7c2777c5 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -206,7 +206,7 @@ where TransportT: Transport + Clone, { let tx_receipt = provider.get_transaction_receipt(*tx_hash).await?; - Ok(tx_receipt.context("Transaction receipt not found.")?) + tx_receipt.context("Transaction receipt not found.") } /// Parse the access list data into a hashmap. diff --git a/zero/src/tracing.rs b/zero/src/tracing.rs index aa404059d..20d376d86 100644 --- a/zero/src/tracing.rs +++ b/zero/src/tracing.rs @@ -5,7 +5,6 @@ pub fn init() { .with( tracing_subscriber::fmt::layer() .with_ansi(false) - .json() .with_filter(EnvFilter::from_default_env()), ) .init(); From 54a7df8416d32c92b986b8fa0e0ab4029e5d89ef Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 9 Oct 2024 23:54:23 +0200 Subject: [PATCH 061/112] feedback: rustdoc --- evm_arithmetization/src/generation/jumpdest.rs | 10 +++++----- zero/src/rpc/jumpdest.rs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index f980d51ac..6def65e77 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -11,14 +11,14 @@ //! Since an operation like e.g. `PUSH 0x5B` does not encode a valid //! [`JUMPDEST`] in its second byte, and `PUSH32 //! 5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B5B` does not -//! encode valid [`JUMPDESTS`] in bytes 1-32, some diligence must be exercised -//! when proving validity of jump operations. +//! encode any valid [`JUMPDEST`] in bytes 1-32, some diligence must be +//! exercised when proving validity of jump operations. //! //! This module concerns itself with data structures for collecting these //! offsets for [`JUMPDEST`] that was visited during an execution and are not -//! recording duplicity. The proofs that each of these offsets are not rendered -//! invalid by any of the previous 32 bytes `PUSH1`-`PUSH32` is computed later -//! in [`prove_context_jumpdests`] on basis of these collections. +//! recording duplicity. The proofs, that each of these offsets are not rendered +//! invalid by `PUSH1`-`PUSH32` in any of the previous 32 bytes, are computed +//! later in `prove_context_jumpdests` on basis of these collections. //! //! [`JUMPDEST`]: https://www.evm.codes/?fork=cancun#5b diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 549506177..0b8016f61 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -347,9 +347,9 @@ fn trace_to_tx_structlog(tx_hash: Option, trace: GethTrace) -> Option Date: Thu, 10 Oct 2024 16:52:30 +0200 Subject: [PATCH 062/112] feedback: add user-specified timeout --- zero/src/bin/leader.rs | 2 ++ zero/src/bin/leader/cli.rs | 5 +++++ zero/src/bin/leader/client.rs | 3 +++ zero/src/bin/rpc.rs | 9 +++++++++ zero/src/parsing.rs | 13 ++++++++++++- zero/src/rpc/jerigon.rs | 13 ++++++++++--- zero/src/rpc/jumpdest.rs | 6 ++---- zero/src/rpc/mod.rs | 5 ++++- zero/src/rpc/native/mod.rs | 7 +++++-- zero/src/rpc/native/txn.rs | 34 +++++++++++++++++++++------------- 10 files changed, 73 insertions(+), 24 deletions(-) diff --git a/zero/src/bin/leader.rs b/zero/src/bin/leader.rs index 775439fde..d1b7c70fd 100644 --- a/zero/src/bin/leader.rs +++ b/zero/src/bin/leader.rs @@ -79,6 +79,7 @@ async fn main() -> Result<()> { block_time, backoff, max_retries, + timeout, } => { let previous_proof = get_previous_proof(previous_proof)?; let block_interval = BlockInterval::new(&block_interval)?; @@ -93,6 +94,7 @@ async fn main() -> Result<()> { max_retries, block_time, jumpdest_src, + timeout, }, block_interval, LeaderConfig { diff --git a/zero/src/bin/leader/cli.rs b/zero/src/bin/leader/cli.rs index dac8c0b41..ede966493 100644 --- a/zero/src/bin/leader/cli.rs +++ b/zero/src/bin/leader/cli.rs @@ -1,7 +1,9 @@ use std::path::PathBuf; +use std::time::Duration; use alloy::transports::http::reqwest::Url; use clap::{Parser, Subcommand, ValueHint}; +use zero::parsing::parse_duration; use zero::prover::cli::CliProverConfig; use zero::prover_state::cli::CliProverStateConfig; use zero::rpc::{JumpdestSrc, RpcType}; @@ -70,6 +72,9 @@ pub(crate) enum Command { /// The maximum number of retries #[arg(long, default_value_t = 0)] max_retries: u32, + /// Timeout for fetching structlog traces + #[arg(long, default_value = "60", value_parser = parse_duration)] + timeout: Duration, }, /// Reads input from HTTP and writes output to a directory. Http { diff --git a/zero/src/bin/leader/client.rs b/zero/src/bin/leader/client.rs index bd01fcdcd..b999bb584 100644 --- a/zero/src/bin/leader/client.rs +++ b/zero/src/bin/leader/client.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::Duration; use alloy::rpc::types::{BlockId, BlockNumberOrTag}; use alloy::transports::http::reqwest::Url; @@ -21,6 +22,7 @@ pub struct RpcParams { pub max_retries: u32, pub block_time: u64, pub jumpdest_src: JumpdestSrc, + pub timeout: Duration, } #[derive(Debug)] @@ -94,6 +96,7 @@ pub(crate) async fn client_main( leader_config.checkpoint_block_number, rpc_params.rpc_type, rpc_params.jumpdest_src, + rpc_params.timeout, ) .await?; block_tx diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index 49c6fa995..e87e9e34d 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::Duration; use alloy::primitives::B256; use alloy::providers::Provider; @@ -13,6 +14,7 @@ use tracing_subscriber::{prelude::*, EnvFilter}; use url::Url; use zero::block_interval::BlockInterval; use zero::block_interval::BlockIntervalStream; +use zero::parsing::parse_duration; use zero::prover::BlockProverInput; use zero::provider::CachedProvider; use zero::rpc; @@ -27,6 +29,7 @@ struct FetchParams { pub checkpoint_block_number: Option, pub rpc_type: RpcType, pub jumpdest_src: JumpdestSrc, + pub timeout: Duration, } #[derive(Args, Clone, Debug)] @@ -51,6 +54,9 @@ struct RpcToolConfig { /// The maximum number of retries. #[arg(long, default_value_t = 0)] max_retries: u32, + /// Timeout for fetching structlog traces + #[arg(long, default_value = "60", value_parser = parse_duration)] + timeout: Duration, } #[derive(Subcommand)] @@ -113,6 +119,7 @@ where checkpoint_block_number, params.rpc_type, params.jumpdest_src, + params.timeout, ) .await?; @@ -142,6 +149,7 @@ impl Cli { checkpoint_block_number, rpc_type: self.config.rpc_type, jumpdest_src: self.config.jumpdest_src, + timeout: self.config.timeout, }; let block_prover_inputs = @@ -169,6 +177,7 @@ impl Cli { checkpoint_block_number: None, rpc_type: self.config.rpc_type, jumpdest_src: self.config.jumpdest_src, + timeout: self.config.timeout, }; let block_prover_inputs = diff --git a/zero/src/parsing.rs b/zero/src/parsing.rs index d1452a464..d22634968 100644 --- a/zero/src/parsing.rs +++ b/zero/src/parsing.rs @@ -1,5 +1,11 @@ //! Parsing utilities. -use std::{fmt::Display, ops::Add, ops::Range, str::FromStr}; +use std::{ + fmt::Display, + num::ParseIntError, + ops::{Add, Range}, + str::FromStr, + time::Duration, +}; use thiserror::Error; @@ -79,6 +85,11 @@ where } } +pub fn parse_duration(arg: &str) -> Result { + let seconds = arg.parse()?; + Ok(Duration::from_secs(seconds)) +} + #[cfg(test)] mod test { use super::*; diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 21480ab4a..6c9f08e5e 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -1,5 +1,6 @@ use core::iter::Iterator; use std::ops::Deref as _; +use std::time::Duration; use alloy::eips::BlockNumberOrTag; use alloy::{ @@ -33,6 +34,7 @@ pub async fn block_prover_input( target_block_id: BlockId, checkpoint_block_number: u64, jumpdest_src: JumpdestSrc, + fetch_timeout: Duration, ) -> anyhow::Result where ProviderT: Provider, @@ -71,6 +73,7 @@ where &block, cached_provider.get_provider().await?.deref(), &tx_results, + &fetch_timeout, ) .await .unwrap_or_else(|e| { @@ -114,14 +117,18 @@ pub async fn process_transactions<'i, ProviderT, TransportT>( block: &Block, provider: &ProviderT, tx_results: &[TxnInfo], + fetch_timeout: &Duration, ) -> anyhow::Result>> where ProviderT: Provider, TransportT: Transport + Clone, { - let block_structlogs = - get_block_normalized_structlogs(provider, &BlockNumberOrTag::from(block.header.number)) - .await?; + let block_structlogs = get_block_normalized_structlogs( + provider, + &BlockNumberOrTag::from(block.header.number), + fetch_timeout, + ) + .await?; let tx_traces = tx_results .iter() diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 0b8016f61..0645254a8 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -28,9 +28,6 @@ use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; use tracing::{trace, warn}; -/// The maximum time we are willing to wait for a block's structlogs before -/// failing over to simulating the JumpDest analysis. -const TIMEOUT_LIMIT: Duration = Duration::from_secs(60); #[derive(Debug, Clone)] pub struct TxStructLogs(pub Option, pub Vec); @@ -60,6 +57,7 @@ fn get_code_hash(usage: &ContractCodeUsage) -> H256 { pub(crate) async fn get_block_normalized_structlogs( provider: &ProviderT, block: &BlockNumberOrTag, + fetch_timeout: &Duration, ) -> anyhow::Result>> where ProviderT: Provider, @@ -69,7 +67,7 @@ where provider.debug_trace_block_by_number(*block, structlog_tracing_options(true, false, false)); let block_stackonly_structlog_traces = - match timeout(TIMEOUT_LIMIT, block_stackonly_structlog_traces_fut).await { + match timeout(*fetch_timeout, block_stackonly_structlog_traces_fut).await { Ok(traces) => traces?, Err(elapsed) => { trace!(target: "fetching block structlogs timed out", ?elapsed); diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index 4e8a0278e..1fa63089d 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -1,6 +1,6 @@ zk_evm_common::check_chain_features!(); -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use __compat_primitive_types::{H256, U256}; use alloy::{ @@ -56,6 +56,7 @@ pub async fn block_prover_input( checkpoint_block_number: u64, rpc_type: RpcType, jumpdest_src: JumpdestSrc, + fetch_timeout: Duration, ) -> Result where ProviderT: Provider, @@ -68,6 +69,7 @@ where block_id, checkpoint_block_number, jumpdest_src, + fetch_timeout, ) .await } @@ -77,6 +79,7 @@ where block_id, checkpoint_block_number, jumpdest_src, + fetch_timeout, ) .await } diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 76dd302ae..5fa2ff0aa 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -1,5 +1,5 @@ -use std::ops::Deref; use std::sync::Arc; +use std::{ops::Deref, time::Duration}; use alloy::{ providers::Provider, @@ -25,13 +25,14 @@ pub async fn block_prover_input( block_number: BlockId, checkpoint_block_number: u64, jumpdest_src: JumpdestSrc, + fetch_timeout: Duration, ) -> anyhow::Result where ProviderT: Provider, TransportT: Transport + Clone, { let (block_trace, other_data) = try_join!( - process_block_trace(provider.clone(), block_number, jumpdest_src), + process_block_trace(provider.clone(), block_number, jumpdest_src, &fetch_timeout), crate::rpc::fetch_other_block_data(provider.clone(), block_number, checkpoint_block_number) )?; @@ -46,6 +47,7 @@ pub(crate) async fn process_block_trace( cached_provider: Arc>, block_number: BlockId, jumpdest_src: JumpdestSrc, + fetch_timeout: &Duration, ) -> anyhow::Result where ProviderT: Provider, @@ -59,6 +61,7 @@ where &block, cached_provider.get_provider().await?.deref(), jumpdest_src, + fetch_timeout, ) .await?; let trie_pre_images = state::process_state_witness(cached_provider, block, &txn_info).await?; diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index a7c2777c5..c9655f543 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -1,5 +1,6 @@ use core::option::Option::None; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::time::Duration; use __compat_primitive_types::{H256, U256}; use alloy::eips::BlockNumberOrTag; @@ -62,6 +63,7 @@ pub async fn process_transactions( block: &Block, provider: &ProviderT, jumpdest_src: JumpdestSrc, + fetch_timeout: &Duration, ) -> anyhow::Result<(CodeDb, Vec)> where ProviderT: Provider, @@ -88,20 +90,26 @@ where JumpdestSrc::ClientFetchedStructlogs => { // In case of the error with retrieving structlogs from the server, // continue without interruption. Equivalent to `ProverSimulation` case. - get_block_normalized_structlogs(provider, &BlockNumberOrTag::from(block.header.number)) - .await - .unwrap_or_else(|e| { - warn!( - "failed to fetch server structlogs for block {}: {e}", - block.header.number - ); - vec![None; block_prestate_trace.len()] - }) - .into_iter() - .map(|tx_struct_log| tx_struct_log.map(|it| it.1)) - .collect() + get_block_normalized_structlogs( + provider, + &BlockNumberOrTag::from(block.header.number), + fetch_timeout, + ) + .await + .unwrap_or_else(|e| { + warn!( + "failed to fetch server structlogs for block {}: {e}", + block.header.number + ); + vec![None; block_prestate_trace.len()] + }) + .into_iter() + .map(|tx_struct_log| tx_struct_log.map(|it| it.1)) + .collect() } - JumpdestSrc::Serverside => todo!(), + JumpdestSrc::Serverside => todo!( + "Not implemented. See https://github.com/0xPolygonZero/erigon/issues/20 for details." + ), }; block From 98b9c8e5c430e537099bd717b16f17a1597444bd Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 11 Oct 2024 17:23:13 +0200 Subject: [PATCH 063/112] feedback --- Cargo.toml | 41 ++++++++-------- .../benches/fibonacci_25m_gas.rs | 2 +- .../src/cpu/kernel/interpreter.rs | 3 +- .../src/cpu/kernel/tests/add11.rs | 4 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 2 +- evm_arithmetization/tests/add11_yml.rs | 2 +- evm_arithmetization/tests/erc20.rs | 2 +- evm_arithmetization/tests/erc721.rs | 2 +- evm_arithmetization/tests/log_opcode.rs | 2 +- evm_arithmetization/tests/selfdestruct.rs | 2 +- evm_arithmetization/tests/simple_transfer.rs | 2 +- evm_arithmetization/tests/withdrawals.rs | 2 +- trace_decoder/src/typed_mpt.rs | 1 - zero/src/rpc/jumpdest.rs | 47 ++++++------------- zero/src/tracing.rs | 1 + 15 files changed, 47 insertions(+), 68 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bfe6ca64c..f3f368650 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,13 @@ [workspace] members = [ - "common", - "compat", - "evm_arithmetization", - "mpt_trie", - "proc_macro", - "smt_trie", - "trace_decoder", - "zero", + "common", + "compat", + "evm_arithmetization", + "mpt_trie", + "proc_macro", + "smt_trie", + "trace_decoder", + "zero", ] resolver = "2" @@ -22,18 +22,18 @@ categories = ["cryptography::cryptocurrencies"] [workspace.dependencies] __compat_primitive_types = { version = "0.12.2", package = "primitive-types" } alloy = { version = '0.3.0', default-features = false, features = [ - "consensus", - "reqwest", - "json-rpc", - "rlp", - "rpc", - "rpc-client", - "rpc-types-eth", - "rpc-types-trace", - "providers", - "transports", - "transport-http", - "rpc-types-debug", + "consensus", + "reqwest", + "json-rpc", + "rlp", + "rpc", + "rpc-client", + "rpc-types-eth", + "rpc-types-trace", + "providers", + "transports", + "transport-http", + "rpc-types-debug", ] } alloy-primitives = "0.8.0" alloy-serde = "0.3.0" @@ -52,7 +52,6 @@ criterion = "0.5.1" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" -enumn = "0.1.13" env_logger = "0.11.3" eth_trie = "0.4.0" ethereum-types = "0.14.1" diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index 26ed8136e..38344629c 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -192,7 +192,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }) } diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 267d393eb..31857ac61 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -60,7 +60,7 @@ pub(crate) struct Interpreter { pub(crate) halt_context: Option, /// Counts the number of appearances of each opcode. For debugging purposes. pub(crate) opcode_count: HashMap, - /// A table of contexts and their reached JUMPDESTs. + /// A table of call contexts and the JUMPDEST offsets that they jumped to. jumpdest_table: HashMap>, /// `true` if the we are currently carrying out a jumpdest analysis. pub(crate) is_jumpdest_analysis: bool, @@ -105,7 +105,6 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - // (interpreter.generation_state.jumpdest_table).map(|x| (x, jdtw)) interpreter.generation_state.jumpdest_table = Some(a.clone()); Some((a, jdtw)) } diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index a71f2d67d..71933ec3e 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -193,7 +193,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let initial_stack = vec![]; @@ -371,7 +371,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index d254a19b1..27374888c 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,7 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 5c955d482..87d959925 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,7 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index b86dcb59f..2cb9a538b 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -195,7 +195,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index ba139f6d2..f34d5c621 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -199,7 +199,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index afceefca3..871fa90f7 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -266,7 +266,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 6a13ace55..eaf56dea6 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -170,7 +170,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index 56c8ba0ae..f40eadc67 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -162,7 +162,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index d2d461030..4b3656cb6 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -105,7 +105,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: Default::default(), + jumpdest_table: None, }; let max_cpu_len_log = 20; diff --git a/trace_decoder/src/typed_mpt.rs b/trace_decoder/src/typed_mpt.rs index f9c1467ab..8baf3cf29 100644 --- a/trace_decoder/src/typed_mpt.rs +++ b/trace_decoder/src/typed_mpt.rs @@ -280,7 +280,6 @@ pub trait StateTrie { ) -> anyhow::Result>; fn insert_hash_by_key(&mut self, key: TrieKey, hash: H256) -> anyhow::Result<()>; fn get_by_address(&self, address: Address) -> Option; - #[allow(dead_code)] fn reporting_remove(&mut self, address: Address) -> anyhow::Result>; /// _Hash out_ parts of the trie that aren't in `txn_ixs`. fn mask(&mut self, address: impl IntoIterator) -> anyhow::Result<()>; diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 0645254a8..1cac95203 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -1,13 +1,10 @@ use core::default::Default; use core::option::Option::None; -use core::str::FromStr as _; use std::collections::HashMap; use std::ops::Not as _; use std::time::Duration; use ::compat::Compat; -use __compat_primitive_types::H160; -use __compat_primitive_types::H256; use alloy::eips::BlockNumberOrTag; use alloy::primitives::Address; use alloy::providers::ext::DebugApi; @@ -22,12 +19,15 @@ use anyhow::bail; use anyhow::ensure; use evm_arithmetization::jumpdest::JumpDestTableWitness; use keccak_hash::keccak; +use ruint::Uint; use tokio::time::timeout; use trace_decoder::is_precompile; use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; use tracing::{trace, warn}; +use crate::rpc::H256; + #[derive(Debug, Clone)] pub struct TxStructLogs(pub Option, pub Vec); @@ -124,9 +124,7 @@ pub(crate) fn generate_jumpdest_table<'a>( ); let entrypoint_code_hash: H256 = match tx.to { - Some(to_address) if is_precompile(H160::from_str(&to_address.to_string())?) => { - return Ok(jumpdest_table) - } + Some(to_address) if is_precompile(to_address.compat()) => return Ok(jumpdest_table), Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { return Ok(jumpdest_table) } @@ -173,17 +171,8 @@ pub(crate) fn generate_jumpdest_table<'a>( ctx, entry.pc, op, - // ?entry, ); - // trace!("TX: {:?}", tx.hash); - // trace!("STEP: {:?}", step); - // trace!("STEPS: {:?}", struct_log.len()); - // trace!("OPCODE: {}", entry.op.as_str()); - // trace!("CODE: {:?}", code_hash); - // trace!("CTX: {:?}", ctx); - // trace!("CURR_DEPTH: {:?}", curr_depth); - // trace!("{:#?}\n", entry); match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { @@ -209,34 +198,19 @@ pub(crate) fn generate_jumpdest_table<'a>( unreachable!() }; - // if *address > U256::from(U160::MAX) { - // trace!( - // "{op}: Callee address {} was larger than possible {}.", - // *address, - // U256::from(U160::MAX) - // ); - // // Se note above. - // continue; - // }; - let a: [u8; 32] = address.compat().into(); - // a <<= 96; - // a >>= 96; - // let aa: [u8; 32] = a.into(); - let aaa: H256 = a.into(); - let lower_20_bytes: [u8; 20] = H160::from(aaa).into(); - let callee_address = Address::from(lower_20_bytes); + let callee_address = stack_value_to_address(address); if callee_addr_to_code_hash.contains_key(&callee_address) { let next_code_hash = callee_addr_to_code_hash[&callee_address]; call_stack.push((next_code_hash, next_ctx_available)); }; - if is_precompile(H160::from_str(&callee_address.to_string())?) { + if is_precompile(callee_address.compat()) { trace!("Called precompile at address {}.", &callee_address); }; if callee_addr_to_code_hash.contains_key(&callee_address).not() - && is_precompile(H160::from_str(&callee_address.to_string())?).not() + && is_precompile(callee_address.compat()).not() { // This case happens if calling an EOA. This is described // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun @@ -329,6 +303,13 @@ pub(crate) fn generate_jumpdest_table<'a>( Ok(jumpdest_table) } +fn stack_value_to_address(operand: &Uint<256, 4>) -> Address { + let all_bytes: [u8; 32] = operand.compat().into(); + let mut lower_20_bytes = [0u8; 20]; + lower_20_bytes[0..20].copy_from_slice(&all_bytes[..]); + Address::from(lower_20_bytes) +} + fn trace_to_tx_structlog(tx_hash: Option, trace: GethTrace) -> Option { match trace { GethTrace::Default(structlog_frame) => { diff --git a/zero/src/tracing.rs b/zero/src/tracing.rs index 20d376d86..6c3f9a8bc 100644 --- a/zero/src/tracing.rs +++ b/zero/src/tracing.rs @@ -5,6 +5,7 @@ pub fn init() { .with( tracing_subscriber::fmt::layer() .with_ansi(false) + .compact() .with_filter(EnvFilter::from_default_env()), ) .init(); From 4707d3853067d1da0329cdc529514185b168f200 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 14:34:37 +0200 Subject: [PATCH 064/112] fix: addresses --- evm_arithmetization/src/generation/prover_input.rs | 2 +- zero/src/rpc/jumpdest.rs | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 1e4090aa8..3a50edf2f 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -394,7 +394,7 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = (*jumpdest_table).get_mut(&context) + if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) && let Some(next_jumpdest_proof) = ctx_jumpdest_table.pop() { Ok(next_jumpdest_proof.into()) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 1cac95203..01543b23b 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -199,6 +199,8 @@ pub(crate) fn generate_jumpdest_table<'a>( }; let callee_address = stack_value_to_address(address); + let callee_address_old = stack_value_to_address_old(address); + assert_eq!(callee_address, callee_address_old); if callee_addr_to_code_hash.contains_key(&callee_address) { let next_code_hash = callee_addr_to_code_hash[&callee_address]; @@ -306,7 +308,17 @@ pub(crate) fn generate_jumpdest_table<'a>( fn stack_value_to_address(operand: &Uint<256, 4>) -> Address { let all_bytes: [u8; 32] = operand.compat().into(); let mut lower_20_bytes = [0u8; 20]; - lower_20_bytes[0..20].copy_from_slice(&all_bytes[..]); + // Based on `__compat_primitive_types::H160::from(H256::from(all_bytes)). + // into()`. + lower_20_bytes[0..20].copy_from_slice(&all_bytes[32 - 20..32]); + Address::from(lower_20_bytes) +} + +// Review: to be removed +fn stack_value_to_address_old(operand: &Uint<256, 4>) -> Address { + let all_bytes: [u8; 32] = operand.compat().into(); + let aaa: H256 = all_bytes.into(); + let lower_20_bytes: [u8; 20] = __compat_primitive_types::H160::from(aaa).into(); Address::from(lower_20_bytes) } From 484350105dda5be922e7f93e09dcf2ad3fccc66f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 14:58:31 +0200 Subject: [PATCH 065/112] todo: fix todo --- trace_decoder/src/core.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 17de5e6a1..6445dbb29 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -184,7 +184,7 @@ pub fn entrypoint( block_hashes: b_hashes.clone(), burn_addr, jumpdest_table: { - // TODO See the issue Simulate to get jumpdests on a per-transaction basis #653. + // TODO(einar-polygon): // Note that this causes any batch containing just a single `None` to collapse // into a `None`, which causing failover to simulating jumpdest analysis for the // whole batch. There is an optimization opportunity here. From 8f980d212f0101bd7cd78f12864c373b7fcaa0d9 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 15:14:36 +0200 Subject: [PATCH 066/112] testing: improve prove_stdio script --- scripts/prove_stdio.sh | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index d5496c63f..f0db105a8 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -50,6 +50,11 @@ if [[ $INPUT_FILE == "" ]]; then exit 1 fi +if [[ ! -s $INPUT_FILE ]]; then + echo "Input file $INPUT_FILE does not exist or has length 0." + exit 1 +fi + # Circuit sizes only matter in non test_only mode. if ! [[ $TEST_ONLY == "test_only" ]]; then if [[ $INPUT_FILE == *"witness_b19807080"* ]]; then @@ -98,30 +103,35 @@ fi # proof. This is useful for quickly testing decoding and all of the # other non-proving code. if [[ $TEST_ONLY == "test_only" ]]; then - cargo run --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR --batch-size $BATCH_SIZE --save-inputs-on-error stdio < $INPUT_FILE |& tee &> $TEST_OUT_PATH + nice -19 cargo run --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR --batch-size $BATCH_SIZE --save-inputs-on-error stdio < $INPUT_FILE |& tee &> $TEST_OUT_PATH if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then echo -e "\n\nSuccess - Note this was just a test, not a proof" #rm $TEST_OUT_PATH exit 0 + elif grep -q 'Attempted to collapse an extension node' $TEST_OUT_PATH; then + echo "ERROR: Attempted to collapse an extension node. See $TEST_OUT_PATH for more details." + exit 4 + elif grep -q 'SIMW == RPCW ? false' $TEST_OUT_PATH; then + echo "ERROR: SIMW == RPCW ? false. See $TEST_OUT_PATH for more details." + exit 5 elif grep -q 'Proving task finished with error' $TEST_OUT_PATH; then # Some error occurred, display the logs and exit. - cat $OUT_LOG_PATH - echo "Failed to create proof witnesses. See $OUT_LOG_PATH for more details." + echo "ERROR: Proving task finished with error. See $TEST_OUT_PATH for more details." exit 1 else - echo -e "\n\nUndecided. Proving process has stopped but verdict is undecided. See \"zk_evm/test.out\" for more details." + echo -e "\n\nUndecided. Proving process has stopped but verdict is undecided. See $TEST_OUT_PATH for more details." exit 2 fi fi cargo build --release --jobs "$num_procs" + start_time=$(date +%s%N) -"${REPO_ROOT}/target/release/leader" --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE \ +nice -19 "${REPO_ROOT}/target/release/leader" --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE \ --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $OUTPUT_LOG end_time=$(date +%s%N) -set +o pipefail cat $OUTPUT_LOG | grep "Successfully wrote to disk proof file " | awk '{print $NF}' | tee $PROOFS_FILE_LIST if [ ! -s "$PROOFS_FILE_LIST" ]; then # Some error occurred, display the logs and exit. @@ -134,7 +144,7 @@ cat $PROOFS_FILE_LIST | while read proof_file; do echo "Verifying proof file $proof_file" verify_file=$PROOF_OUTPUT_DIR/verify_$(basename $proof_file).out - "${REPO_ROOT}/target/release/verifier" -f $proof_file | tee $verify_file + nice -19 "${REPO_ROOT}/target/release/verifier" -f $proof_file | tee $verify_file if grep -q 'All proofs verified successfully!' $verify_file; then echo "Proof verification for file $proof_file successful"; rm $verify_file # we keep the generated proof for potential reuse From ee7e5f303a18b67b68de0e70c506921025dcffde Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 15:15:44 +0200 Subject: [PATCH 067/112] testing: improve test_native script --- scripts/test_native.sh | 104 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 96 insertions(+), 8 deletions(-) diff --git a/scripts/test_native.sh b/scripts/test_native.sh index eb65b5758..3093735b8 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -48,7 +48,94 @@ ROUND2=" 20727650 " +ROUND3=" +20727643 +20727644 +20727648 +20727649 +20727650 +" + +ROUND4=" +19457111 +19477724 +19501672 +19508907 +19511272 +19548904 +19550401 +19553425 +19563122 +19585193 +19600168 +19603017 +19607029 +19649976 +19654474 +19657021 +19670735 +19688239 +19737540 +19767306 +19792995 +19812505 +19829370 +19835094 +19862390 +19871215 +19877263 +19877279 +19893964 +19922838 +19938970 +19971875 +20011069 +20071977 +20131326 +20173673 +20182890 +20218660 +20225259 +20229861 +20259823 +20274215 +20288828 +20291090 +20301243 +20346949 +20410573 +20462322 +20518465 +20521004 +20542632 +20543651 +20555951 +20634148 +20691605 +20714397 +20715309 +20715461 +20719386 +20720179 +20720275 +20741147 +20775888 +20804319 +20835783 +20859523 +20727643 +20727644 +20727648 +20727649 +20727650 +" +ROUND5=" +19650385 +19542391 +19578175 +19511272 +" CANCUN=19426587 @@ -60,27 +147,28 @@ RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." -#BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS" -BLOCKS="20727641" -#BLOCKS="$CANCUNBLOCKS" +#BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" +#BLOCKS="19511272" +BLOCKS=$RANDOMBLOCKS +BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" echo "Testing: $BLOCKS" -printf "\n\ngithash block verdict duration\n" | tee -a witnesses/native_results.txt -echo "------------------------------------" | tee -a witnesses/native_results.txt +printf "\n\ngithash block verdict r duration\n" | tee -a witnesses/native_results.txt +echo "----------------------------------------" | tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace - cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + nice -19 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS rg "jump" $WITNESS echo "Testing blocks: $BLOCKS." echo "Now testing block $BLOCK .." export RUST_LOG=info SECONDS=0 - timeout 10m ./prove_stdio.sh $WITNESS test_only $BLOCK + timeout 10m nice -19 ./prove_stdio.sh $WITNESS test_only $BLOCK EXITCODE=$? DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` echo $DURATION @@ -90,5 +178,5 @@ for BLOCK in $BLOCKS; do else VERDICT="failure" fi - printf "%s %10i %s %s\n" $GITHASH $BLOCK $VERDICT $DURATION | tee -a witnesses/native_results.txt + printf "%s %10i %s %3i %s\n" $GITHASH $BLOCK $VERDICT $EXITCODE $DURATION | tee -a witnesses/native_results.txt done From 5451399e4f1866e3fd2d0af1a6c87d4cfd7e5cef Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 16:51:30 +0200 Subject: [PATCH 068/112] fmt --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f3f368650..66050ed4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,4 +123,4 @@ plonky2_util = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f2b06500e16ad4d7f1c2b057903602eed" } [workspace.lints.clippy] -too_long_first_doc_paragraph = "allow" \ No newline at end of file +too_long_first_doc_paragraph = "allow" From e9a8702a64ee49020f9085f4f3e5a7e16a83bc32 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 14 Oct 2024 17:50:41 +0200 Subject: [PATCH 069/112] Round 5 --- .cargo/config.toml | 13 ++++++++++++- Cargo.toml | 24 ++++++++++++++++++++++++ scripts/prove_stdio.sh | 2 +- scripts/test_native.sh | 2 +- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 6340ce34a..526c1c9b4 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,4 +1,15 @@ [build] # https://github.com/rust-lang/rust/pull/124129 # https://github.com/dtolnay/linkme/pull/88 -rustflags = ["-Z", "linker-features=-lld"] + +[env] +RUST_BACKTRACE = "1" +RUST_TEST_NOCAPTURE = "1" + +[term] +verbose = true +color = 'auto' + +[target.x86_64-unknown-linux-gnu] +linker = "clang" +rustflags = ["-Z", "linker-features=-lld", "-C", "target-cpu=native"] #, "-C", "link-arg=-fuse-ld=/usr/bin/mold", "-C", "debuginfo=2"] diff --git a/Cargo.toml b/Cargo.toml index 66050ed4b..5fb943015 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,3 +124,27 @@ starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" + +[profile.release] +opt-level = 3 +debug = true +incremental = true +debug-assertions = true +lto = false +overflow-checks = false + +[profile.test] +opt-level = 3 +debug = true +incremental = true +debug-assertions = true +lto = false +overflow-checks = false + +[profile.dev] +opt-level = 3 +debug = true +incremental = true +debug-assertions = true +lto = false +overflow-checks = false diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 48e5ecf6a..18ef39b40 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -1,6 +1,6 @@ #!/bin/bash # ------------------------------------------------------------------------------ -set -exo pipefail +set -ex # Run prover with the parsed input from the standard terminal. # To generate the json input file, use the `rpc` tool, for example: diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 3093735b8..d08446d4d 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -149,7 +149,7 @@ echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" #BLOCKS="19511272" -BLOCKS=$RANDOMBLOCKS +BLOCKS=$ROUND5 BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" From b2f66edeedac265b1a896f0e47c4aeca5c35e477 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 15 Oct 2024 09:23:51 +0200 Subject: [PATCH 070/112] testing --- scripts/prove_stdio.sh | 1 + scripts/test_native.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 18ef39b40..98daf1098 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -110,6 +110,7 @@ if [[ $TEST_ONLY == "test_only" ]]; then exit 0 elif grep -q 'Attempted to collapse an extension node' $TEST_OUT_PATH; then echo "ERROR: Attempted to collapse an extension node. See $TEST_OUT_PATH for more details." + rm $TEST_OUT_PATH exit 4 elif grep -q 'SIMW == RPCW ? false' $TEST_OUT_PATH; then echo "ERROR: SIMW == RPCW ? false. See $TEST_OUT_PATH for more details." diff --git a/scripts/test_native.sh b/scripts/test_native.sh index d08446d4d..fb97f767b 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -141,7 +141,7 @@ ROUND5=" CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` STATICTIP=20721266 -NUMRANDOMBLOCKS=100 +NUMRANDOMBLOCKS=1000 RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` GITHASH=`git rev-parse --short HEAD` @@ -149,7 +149,7 @@ echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" #BLOCKS="19511272" -BLOCKS=$ROUND5 +BLOCKS="$RANDOMBLOCKS" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" From cfb293c8bb3d4ec0b9e9f3bb0e00881311959d74 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 15 Oct 2024 20:02:33 +0200 Subject: [PATCH 071/112] testing: improve reporting, add error cases --- evm_arithmetization/src/generation/state.rs | 4 +- scripts/test_native.sh | 87 +++++++++++++++++++-- 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index d44b7dea4..d8cc3e93b 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -486,12 +486,12 @@ impl GenerationState { // We cannot observe anything as the stack is empty. return Ok(()); } - if dst == KERNEL.global_labels["observe_new_address"] { + if dst == KERNEL.global_labels["observe_new_address"] && self.is_kernel() { let tip_u256 = stack_peek(self, 0)?; let tip_h256 = H256::from_uint(&tip_u256); let tip_h160 = H160::from(tip_h256); self.observe_address(tip_h160); - } else if dst == KERNEL.global_labels["observe_new_contract"] { + } else if dst == KERNEL.global_labels["observe_new_contract"] && self.is_kernel() { let tip_u256 = stack_peek(self, 0)?; let tip_h256 = H256::from_uint(&tip_u256); self.observe_contract(tip_h256)?; diff --git a/scripts/test_native.sh b/scripts/test_native.sh index fb97f767b..6a6b6dabe 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -137,6 +137,71 @@ ROUND5=" 19511272 " +ROUND6=" +19426872 +19427018 +19427388 +19427472 +19429634 +19430273 +19430687 +19430855 +19431223 +19431344 +19432360 +19432641 +19435607 +19435804 +19436307 +19439155 +19439754 +19440665 +19441789 +19443628 +19443673 +19444327 +19444582 +19445175 +19445286 +19445799 +19446774 +19446911 +19447598 +19447814 +19448687 +19449229 +19449755 +19450491 +19451118 +19451955 +19452325 +19452532 +19452795 +19452869 +19454136 +19455621 +19456052 +19456615 +19460281 +19460945 +19462377 +19463186 +19464727 +19466034 +19466036 +19466108 +19466509 +" + +ROUND7=" +19430273 +19431344 +19451118 +19452869 +19460945 +19464727 +19466034 +" CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` @@ -149,34 +214,40 @@ echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" #BLOCKS="19511272" -BLOCKS="$RANDOMBLOCKS" +BLOCKS="$ROUND7" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" echo "Testing: $BLOCKS" -printf "\n\ngithash block verdict r duration\n" | tee -a witnesses/native_results.txt -echo "----------------------------------------" | tee -a witnesses/native_results.txt +printf "\n\ngithash block verdict r rpc-time test-time total-time tx-ok tx-none tx-total \n" | tee -a witnesses/native_results.txt +echo "---------------------------------------------------------------------------------------" | tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do + TOTALTIME=0 GITHASH=`git rev-parse --short HEAD` WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace + SECONDS=0 nice -19 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - rg "jump" $WITNESS - echo "Testing blocks: $BLOCKS." + TOTALTIME=`echo -n $(($TOTALTIME + $SECONDS))` + DURATION_RPC=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + TXALL=`grep '"jumpdest_table":' $WITNESS | wc -l` + TXNONE=`grep '"jumpdest_table": null' $WITNESS | wc -l` + TXOK=`echo -n $(($TXALL - $TXNONE))` echo "Now testing block $BLOCK .." export RUST_LOG=info SECONDS=0 timeout 10m nice -19 ./prove_stdio.sh $WITNESS test_only $BLOCK EXITCODE=$? - DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` - echo $DURATION + TOTALTIME=`echo -n $(($TOTALTIME + $SECONDS))` + DURATION_PRV=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` + TOTALTIME=`date -u -d @"$TOTALTIME" +'%-Hh%-Mm%-Ss'` if [ $EXITCODE -eq 0 ] then VERDICT="success" else VERDICT="failure" fi - printf "%s %10i %s %3i %s\n" $GITHASH $BLOCK $VERDICT $EXITCODE $DURATION | tee -a witnesses/native_results.txt + printf "%s %10i %s %3i %8s %8s %8s %3i %3i %3i \n" $GITHASH $BLOCK $VERDICT $EXITCODE $DURATION_RPC $DURATION_PRV $TOTALTIME $TXOK $TXNONE $TXALL | tee -a witnesses/native_results.txt done From 3c497cc9b5dfc506e900f835b511f6149f4b2c7e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 15 Oct 2024 21:15:14 +0200 Subject: [PATCH 072/112] change exit code --- scripts/prove_stdio.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 98daf1098..5ee39f296 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -52,7 +52,7 @@ fi if [[ ! -s $INPUT_FILE ]]; then echo "Input file $INPUT_FILE does not exist or has length 0." - exit 1 + exit 6 fi # Circuit sizes only matter in non test_only mode. From 2dc52cbc4c2671cd3656e2ea5a033da4ed52786c Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 16 Oct 2024 16:17:33 +0200 Subject: [PATCH 073/112] don't panic! --- evm_arithmetization/src/generation/prover_input.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 3a50edf2f..bfbea41b2 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -359,10 +359,10 @@ impl GenerationState { } /// Returns the next used jump address. - /// todo fn run_next_jumpdest_table_address(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; + // TODO(einar-polygon) // get_code from self.memory if self.jumpdest_table.is_none() { @@ -824,7 +824,7 @@ impl GenerationState { } info!("SIMW == RPCW ? {}", simw == &rpcw); info!("tx: {:?}", self.inputs.txn_hashes); - panic!(); + // panic!(); // info!("SIMP {:?}", &simp); // info!("RPCP {:?}", &rpcp); // info!("SIMP == RPCP ? {}", &simp == &rpcp); From dd89251d10421646f9ae3dbf18cc0d5e872e74aa Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 18 Oct 2024 13:00:00 +0200 Subject: [PATCH 074/112] fix type 5 errors --- evm_arithmetization/src/witness/transition.rs | 16 +++++----- scripts/prove_stdio.sh | 2 +- scripts/test_native.sh | 31 +++++++++++++++---- zero/src/rpc/jumpdest.rs | 5 ++- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/evm_arithmetization/src/witness/transition.rs b/evm_arithmetization/src/witness/transition.rs index fdcf9af65..8a6f1d39a 100644 --- a/evm_arithmetization/src/witness/transition.rs +++ b/evm_arithmetization/src/witness/transition.rs @@ -345,14 +345,6 @@ where where Self: Sized, { - self.perform_op(op, row)?; - self.incr_pc(match op { - Operation::Syscall(_, _, _) | Operation::ExitKernel => 0, - Operation::Push(n) => n as usize + 1, - Operation::Jump | Operation::Jumpi => 0, - _ => 1, - }); - self.incr_gas(gas_to_charge(op)); let registers = self.get_registers(); let gas_limit_address = MemoryAddress::new( @@ -373,6 +365,14 @@ where } } + self.perform_op(op, row)?; + self.incr_pc(match op { + Operation::Syscall(_, _, _) | Operation::ExitKernel => 0, + Operation::Push(n) => n as usize + 1, + Operation::Jump | Operation::Jumpi => 0, + _ => 1, + }); + Ok(op) } diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh index 5ee39f296..1e432c8f7 100755 --- a/scripts/prove_stdio.sh +++ b/scripts/prove_stdio.sh @@ -1,6 +1,6 @@ #!/bin/bash # ------------------------------------------------------------------------------ -set -ex +set -x # Run prover with the parsed input from the standard terminal. # To generate the json input file, use the `rpc` tool, for example: diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 6a6b6dabe..142687cfa 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -203,24 +203,41 @@ ROUND7=" 19466034 " +ROUND8=" +19657436 +19508991 +19500774 +19794433 +" + CANCUN=19426587 TIP=`cast block-number --rpc-url $RPC` -STATICTIP=20721266 +STATICTIP=20978815 NUMRANDOMBLOCKS=1000 RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` +REPO_ROOT=$(git rev-parse --show-toplevel) + GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" -#BLOCKS="19511272" -BLOCKS="$ROUND7" +#BLOCKS="$RANDOMBLOCKS" +BLOCKS="$ROUND8" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" echo "Testing: $BLOCKS" -printf "\n\ngithash block verdict r rpc-time test-time total-time tx-ok tx-none tx-total \n" | tee -a witnesses/native_results.txt -echo "---------------------------------------------------------------------------------------" | tee -a witnesses/native_results.txt + +printf "\n\nr\n" | tee -a witnesses/native_results.txt +echo "0 is success" | tee -a witnesses/native_results.txt +echo "5 [defect] is non-matching jumpdest tables" | tee -a witnesses/native_results.txt +echo "1 [unexpected] is other errors" | tee -a witnesses/native_results.txt +echo "4 [expected] is Attempted to collapse an extension node" | tee -a witnesses/native_results.txt +echo "6 [expected] is empty witness. Usually due to Error: Failed to get proof for account" | tee -a witnesses/native_results.txt +echo "Report started: $(date)" | tee -a witnesses/native_results.txt +printf "\ngithash block verdict r rpc-time test-time total-time tx-ok tx-none tx-total \n" | tee -a witnesses/native_results.txt +echo "---------------------------------------------------------------------------------------" | tee -a witnesses/native_results.txt for BLOCK in $BLOCKS; do TOTALTIME=0 @@ -229,12 +246,14 @@ for BLOCK in $BLOCKS; do echo "Fetching block $BLOCK" export RUST_LOG=rpc=trace SECONDS=0 - nice -19 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS + nice -19 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs --timeout 600 fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS TOTALTIME=`echo -n $(($TOTALTIME + $SECONDS))` DURATION_RPC=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` TXALL=`grep '"jumpdest_table":' $WITNESS | wc -l` TXNONE=`grep '"jumpdest_table": null' $WITNESS | wc -l` TXOK=`echo -n $(($TXALL - $TXNONE))` + TEST_OUT_PATH="${REPO_ROOT}/$BLOCK.test.out" + #rm $TEST_OUT_PATH echo "Now testing block $BLOCK .." export RUST_LOG=info SECONDS=0 diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 01543b23b..dcd2d9f81 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -169,7 +169,10 @@ pub(crate) fn generate_jumpdest_table<'a>( tx_hash = ?tx.hash, ?code_hash, ctx, - entry.pc, + pc = entry.pc, + pc_hex = format!("{:08x?}", entry.pc), + gas = entry.gas, + gas_cost = entry.gas_cost, op, ?entry, ); From 6c59c41d99aee7ab8cebce796bbe97713b88a197 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sat, 19 Oct 2024 02:20:25 +0200 Subject: [PATCH 075/112] Fix: 19548491 --- zero/src/rpc/jumpdest.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index dcd2d9f81..368c7e760 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -94,7 +94,7 @@ where /// using a Geth structlog as input. pub(crate) fn generate_jumpdest_table<'a>( tx: &Transaction, - struct_log: &[StructLog], + structlog: &[StructLog], tx_traces: impl Iterator, ) -> anyhow::Result { trace!("Generating JUMPDEST table for tx: {}", tx.hash); @@ -146,7 +146,8 @@ pub(crate) fn generate_jumpdest_table<'a>( let mut call_stack = vec![(entrypoint_code_hash, next_ctx_available)]; next_ctx_available += 1; - for (step, entry) in struct_log.iter().enumerate() { + let mut stuctlog_iter = structlog.iter().enumerate().peekable(); + while let Some((step, entry)) = stuctlog_iter.next() { let op = entry.op.as_str(); let curr_depth: usize = entry.depth.try_into().unwrap(); @@ -169,6 +170,7 @@ pub(crate) fn generate_jumpdest_table<'a>( tx_hash = ?tx.hash, ?code_hash, ctx, + next_ctx_available, pc = entry.pc, pc_hex = format!("{:08x?}", entry.pc), gas = entry.gas, @@ -224,6 +226,19 @@ pub(crate) fn generate_jumpdest_table<'a>( &callee_address ); } + + if let Some((_next_step, next_entry)) = &stuctlog_iter.peek() { + let next_depth: usize = next_entry.depth.try_into().unwrap(); + if next_depth < curr_depth { + // The call caused an exception. Skip over incrementing `next_ctx_available`. + continue; + } + } + // `peek()` only returns `None` if we are at the last entry of + // the Structlog, whether we are on a `CALL` op that throws an + // exception or not. But this is of no consequence to the + // generated Jumpdest table, so we can ignore the case. + next_ctx_available += 1; } "CREATE" | "CREATE2" => { From 0d7f6b7e809221c67837ab6c6d0a48ba03213955 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Sat, 19 Oct 2024 21:54:41 +0200 Subject: [PATCH 076/112] add stats --- scripts/test_native.sh | 57 +++++++++++++++++++++++++++++++++++++--- zero/src/rpc/jumpdest.rs | 5 ++-- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/scripts/test_native.sh b/scripts/test_native.sh index 142687cfa..824001094 100755 --- a/scripts/test_native.sh +++ b/scripts/test_native.sh @@ -6,8 +6,51 @@ if [ -z $RPC ]; then # You must set an RPC endpoint exit 1 fi + +if [ git diff --quiet --exit-code HEAD ]; then + exit 1 +fi + + mkdir -p witnesses + + +RESULT_LEN=$(cat witnesses/native_results.txt | wc -l) + + +function statistics() +{ + PREFIX_LEN=$(($RESULT_LEN + 13)) + wc -l witnesses/native_results.txt + cat witnesses/native_results.txt | tail -n +$PREFIX_LEN + + SUMOK=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f8 | paste -s -d+ - | bc) + SUMFAIL=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f9 | paste -s -d+ - | bc) + SUMTOTAL=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f10 | paste -s -d+ - | bc) + echo $SUMTOTAL + echo $SUMFAIL + echo "Failure rate: " $([[ $SUMTOTAL -eq 0 ]] && echo "0" || echo "$(($SUMFAIL * 100 / $SUMTOTAL))%") + echo "Success rate: " $([[ $SUMTOTAL -eq 0 ]] && echo "0" || echo "$(($SUMOK * 100 / $SUMTOTAL))%") + + ZEROES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "0") + ONES=$( cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "1") + TWOS=$( cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "2") + THREES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "3") + FOURS=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "4") + FIVES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "5") + SIXES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "6") + echo $ZEROES + echo $ONES + echo $TWOS + echo $THREES + echo $FOURS + echo $FIVES + echo $SIXES + echo "good bye" + exit 0 +} +trap statistics INT # EXIT QUIT HUP TERM # Must match the values in prove_stdio.sh or build is dirty. #export RAYON_NUM_THREADS=1 #export TOKIO_WORKER_THREADS=1 @@ -222,8 +265,8 @@ GITHASH=`git rev-parse --short HEAD` echo "Testing against mainnet, current revision: $GITHASH." #BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" -#BLOCKS="$RANDOMBLOCKS" -BLOCKS="$ROUND8" +BLOCKS="$RANDOMBLOCKS" +#BLOCKS="$ROUND8" BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` echo "Testing blocks: $BLOCKS" @@ -252,8 +295,6 @@ for BLOCK in $BLOCKS; do TXALL=`grep '"jumpdest_table":' $WITNESS | wc -l` TXNONE=`grep '"jumpdest_table": null' $WITNESS | wc -l` TXOK=`echo -n $(($TXALL - $TXNONE))` - TEST_OUT_PATH="${REPO_ROOT}/$BLOCK.test.out" - #rm $TEST_OUT_PATH echo "Now testing block $BLOCK .." export RUST_LOG=info SECONDS=0 @@ -269,4 +310,12 @@ for BLOCK in $BLOCKS; do VERDICT="failure" fi printf "%s %10i %s %3i %8s %8s %8s %3i %3i %3i \n" $GITHASH $BLOCK $VERDICT $EXITCODE $DURATION_RPC $DURATION_PRV $TOTALTIME $TXOK $TXNONE $TXALL | tee -a witnesses/native_results.txt + + + ### Clean up + TEST_OUT_PATH="${REPO_ROOT}/$BLOCK.test.out" + rm $TEST_OUT_PATH + rm $WITNESS + done + diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 368c7e760..250513c82 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -227,10 +227,11 @@ pub(crate) fn generate_jumpdest_table<'a>( ); } - if let Some((_next_step, next_entry)) = &stuctlog_iter.peek() { + if let Some((_next_step, next_entry)) = stuctlog_iter.peek() { let next_depth: usize = next_entry.depth.try_into().unwrap(); if next_depth < curr_depth { - // The call caused an exception. Skip over incrementing `next_ctx_available`. + // The call caused an exception. Skip over incrementing + // `next_ctx_available`. continue; } } From b8cf325d655f249c9056c95d507c586419c5a0bd Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 11:06:44 +0200 Subject: [PATCH 077/112] dbg --- evm_arithmetization/src/generation/prover_input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index bfbea41b2..b5018395a 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -829,7 +829,7 @@ impl GenerationState { // info!("RPCP {:?}", &rpcp); // info!("SIMP == RPCP ? {}", &simp == &rpcp); } else { - info!("JUMPDEST tables similar"); + info!("JUMPDEST tables are equal."); } self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; From e9ec9f86bcefe5518afac816714c892223b62ca6 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 11:10:00 +0200 Subject: [PATCH 078/112] remove test scripts --- scripts/test_jerigon.sh | 329 ---------------------------------------- scripts/test_native.sh | 321 --------------------------------------- 2 files changed, 650 deletions(-) delete mode 100755 scripts/test_jerigon.sh delete mode 100755 scripts/test_native.sh diff --git a/scripts/test_jerigon.sh b/scripts/test_jerigon.sh deleted file mode 100755 index 8a515f05e..000000000 --- a/scripts/test_jerigon.sh +++ /dev/null @@ -1,329 +0,0 @@ -#!/usr/bin/env bash - -set -uo pipefail - -RPC=${RPC_JERIGON} -if [ -z $RPC ]; then - # You must set an RPC endpoint - exit 1 -fi -mkdir -p witnesses - -# Must match the values in prove_stdio.sh or build is dirty. -export RAYON_NUM_THREADS=1 -export TOKIO_WORKER_THREADS=1 -export RUST_BACKTRACE=full -#export RUST_LOG=info -#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' -#export RUST_MIN_STACK=67108864 - -GITHASH=`git rev-parse --short HEAD` -echo "Testing against jergion, current revision: $GITHASH." - -CIBLOCKS=" -1 -2 -3 -4 -5 -6 -7 -8 -9 -10 -11 -12 -13 -14 -15 -16 -" - - -STILLFAIL=" -37 -75 -15 -35 -43 -72 -77 -184 -460 -461 -462 -463 -464 -465 -467 -468 -474 -475 -476 -566 -662 -664 -665 -667 -670 -477 -478 -444 -" - -JUMPI=" -662 -664 -665 -667 -670 -" - -CONTAINSKEY=" -461 -462 -463 -464 -465 -467 -468 -474 -475 -476 -72 -" - -CREATE2=" -43 -566 -77 -" - -DECODING=" -477 -478 -" - -USEDTOFAIL=" -2 -15 -28 -35 -37 -43 -65 - -28 - -43 -460 -461 -462 -463 -464 -465 -467 -468 -474 -475 -476 -566 -662 -664 -665 -667 -670 -72 -77 -" - -ROUND2=" -664 -667 -670 -665 -" - -NOWSUCCESS=" -444 -4 -5 -28 -65 -566 -15 -35 -" - -ROUND3=" -125 -127 -131 -132 -136 -141 -142 -143 -145 -149 -150 -151 -153 -154 -186 -187 -188 -190 -193 -195 -197 -199 -201 -214 -220 -221 -222 -223 -226 -228 -229 -230 -231 -232 -234 -242 -256 -257 -258 -262 -264 -267 -268 -282 -284 -285 -287 -292 -294 -295 -301 -303 -304 -321 -325 -333 -460 -461 -462 -463 -464 -465 -466 -467 -468 -473 -474 -528 -529 -530 -531 -532 -533 -534 -566 -570 -664 -77 -548 -" - -ROUND4=" -136 -186 -268 -282 -301 -304 -321 -333 -460 -461 -462 -463 -464 -465 -466 -467 -468 -473 -474 -528 -529 -530 -531 -532 -533 -534 -570 -664 -" - -ROUND5=" -460 -461 -462 -463 -464 -465 -466 -467 -468 -473 -474 -664 -" - -ROUND6=" -664 -" - -# 470..663 from Robin -for i in {470..663} -do - ROBIN+=" $i" -done - -TIP=688 -NUMRANDOMBLOCKS=10 -RANDOMBLOCKS=`shuf --input-range=0-$TIP -n $NUMRANDOMBLOCKS | sort` - -#BLOCKS="72 185" #$ROUND5 $CREATE2 $DECODING $CONTAINSKEY $USEDTOFAIL $STILLFAIL $CIBLOCKS $JUMPI $ROUND2 $RANDOMBLOCKS $ROUND3 $ROUND5 $ROUND4" -BLOCKS="$CIBLOCKS" -BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` - -echo "Testing: $BLOCKS" -printf "\ngithash block verdict duration\n" | tee -a witnesses/jerigon_results.txt -echo "------------------------------------" | tee -a witnesses/jerigon_results.txt - -for BLOCK in $BLOCKS; do - GITHASH=`git rev-parse --short HEAD` - WITNESS="witnesses/$BLOCK.jerigon.$GITHASH.witness.json" - echo "Fetching block $BLOCK" - export RUST_LOG=rpc=trace - SECONDS=0 - cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type jerigon --jumpdest-src client-fetched-structlogs fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - echo "Testing blocks: $BLOCKS." - echo "Now testing block $BLOCK .." - export RUST_LOG=info - timeout 10m ./prove_stdio.sh $WITNESS test_only $BLOCK - EXITCODE=$? - DURATION=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` - echo $DURATION - if [ $EXITCODE -eq 0 ] - then - VERDICT="success" - else - VERDICT="failure" - fi - printf "%s %10i %s %s\n" $GITHASH $BLOCK $VERDICT $DURATION | tee -a witnesses/jerigon_results.txt -done diff --git a/scripts/test_native.sh b/scripts/test_native.sh deleted file mode 100755 index 824001094..000000000 --- a/scripts/test_native.sh +++ /dev/null @@ -1,321 +0,0 @@ -#!/usr/bin/env bash - -set -uxo pipefail - -if [ -z $RPC ]; then - # You must set an RPC endpoint - exit 1 -fi - -if [ git diff --quiet --exit-code HEAD ]; then - exit 1 -fi - - -mkdir -p witnesses - - - -RESULT_LEN=$(cat witnesses/native_results.txt | wc -l) - - -function statistics() -{ - PREFIX_LEN=$(($RESULT_LEN + 13)) - wc -l witnesses/native_results.txt - cat witnesses/native_results.txt | tail -n +$PREFIX_LEN - - SUMOK=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f8 | paste -s -d+ - | bc) - SUMFAIL=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f9 | paste -s -d+ - | bc) - SUMTOTAL=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f10 | paste -s -d+ - | bc) - echo $SUMTOTAL - echo $SUMFAIL - echo "Failure rate: " $([[ $SUMTOTAL -eq 0 ]] && echo "0" || echo "$(($SUMFAIL * 100 / $SUMTOTAL))%") - echo "Success rate: " $([[ $SUMTOTAL -eq 0 ]] && echo "0" || echo "$(($SUMOK * 100 / $SUMTOTAL))%") - - ZEROES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "0") - ONES=$( cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "1") - TWOS=$( cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "2") - THREES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "3") - FOURS=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "4") - FIVES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "5") - SIXES=$(cat witnesses/native_results.txt | tail -n +$PREFIX_LEN | tr -s ' ' | cut -d' ' -f4 | grep --count "6") - echo $ZEROES - echo $ONES - echo $TWOS - echo $THREES - echo $FOURS - echo $FIVES - echo $SIXES - echo "good bye" - exit 0 -} -trap statistics INT # EXIT QUIT HUP TERM -# Must match the values in prove_stdio.sh or build is dirty. -#export RAYON_NUM_THREADS=1 -#export TOKIO_WORKER_THREADS=1 -export RUST_BACKTRACE=full -#export RUST_LOG=info -#export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' -#export RUST_MIN_STACK=33554432 - - - -CANCUNBLOCKS=" -20548415 -20240058 -19665756 -20634472 -19807080 -20634403 -" - -PRECANCUN=" -19096840 -19240700 -" - - -#It's visible with block 20727641 -ROUND1=`echo {20727640..20727650}` - -ROUND2=" -20727641 -20727643 -20727644 -20727645 -20727646 -20727647 -20727648 -20727649 -20727650 -" - -ROUND3=" -20727643 -20727644 -20727648 -20727649 -20727650 -" - -ROUND4=" -19457111 -19477724 -19501672 -19508907 -19511272 -19548904 -19550401 -19553425 -19563122 -19585193 -19600168 -19603017 -19607029 -19649976 -19654474 -19657021 -19670735 -19688239 -19737540 -19767306 -19792995 -19812505 -19829370 -19835094 -19862390 -19871215 -19877263 -19877279 -19893964 -19922838 -19938970 -19971875 -20011069 -20071977 -20131326 -20173673 -20182890 -20218660 -20225259 -20229861 -20259823 -20274215 -20288828 -20291090 -20301243 -20346949 -20410573 -20462322 -20518465 -20521004 -20542632 -20543651 -20555951 -20634148 -20691605 -20714397 -20715309 -20715461 -20719386 -20720179 -20720275 -20741147 -20775888 -20804319 -20835783 -20859523 -20727643 -20727644 -20727648 -20727649 -20727650 -" - -ROUND5=" -19650385 -19542391 -19578175 -19511272 -" - -ROUND6=" -19426872 -19427018 -19427388 -19427472 -19429634 -19430273 -19430687 -19430855 -19431223 -19431344 -19432360 -19432641 -19435607 -19435804 -19436307 -19439155 -19439754 -19440665 -19441789 -19443628 -19443673 -19444327 -19444582 -19445175 -19445286 -19445799 -19446774 -19446911 -19447598 -19447814 -19448687 -19449229 -19449755 -19450491 -19451118 -19451955 -19452325 -19452532 -19452795 -19452869 -19454136 -19455621 -19456052 -19456615 -19460281 -19460945 -19462377 -19463186 -19464727 -19466034 -19466036 -19466108 -19466509 -" - -ROUND7=" -19430273 -19431344 -19451118 -19452869 -19460945 -19464727 -19466034 -" - -ROUND8=" -19657436 -19508991 -19500774 -19794433 -" - -CANCUN=19426587 -TIP=`cast block-number --rpc-url $RPC` -STATICTIP=20978815 -NUMRANDOMBLOCKS=1000 -RANDOMBLOCKS=`shuf --input-range=$CANCUN-$TIP -n $NUMRANDOMBLOCKS | sort` - -REPO_ROOT=$(git rev-parse --show-toplevel) - -GITHASH=`git rev-parse --short HEAD` -echo "Testing against mainnet, current revision: $GITHASH." - -#BLOCKS="$CANCUNBLOCKS $RANDOMBLOCKS $ROUND3" -BLOCKS="$RANDOMBLOCKS" -#BLOCKS="$ROUND8" -BLOCKS=`echo $BLOCKS | tr ' ' '\n' | sort -nu | tr '\n' ' '` -echo "Testing blocks: $BLOCKS" - -echo "Testing: $BLOCKS" - -printf "\n\nr\n" | tee -a witnesses/native_results.txt -echo "0 is success" | tee -a witnesses/native_results.txt -echo "5 [defect] is non-matching jumpdest tables" | tee -a witnesses/native_results.txt -echo "1 [unexpected] is other errors" | tee -a witnesses/native_results.txt -echo "4 [expected] is Attempted to collapse an extension node" | tee -a witnesses/native_results.txt -echo "6 [expected] is empty witness. Usually due to Error: Failed to get proof for account" | tee -a witnesses/native_results.txt -echo "Report started: $(date)" | tee -a witnesses/native_results.txt -printf "\ngithash block verdict r rpc-time test-time total-time tx-ok tx-none tx-total \n" | tee -a witnesses/native_results.txt -echo "---------------------------------------------------------------------------------------" | tee -a witnesses/native_results.txt - -for BLOCK in $BLOCKS; do - TOTALTIME=0 - GITHASH=`git rev-parse --short HEAD` - WITNESS="witnesses/$BLOCK.native.$GITHASH.witness.json" - echo "Fetching block $BLOCK" - export RUST_LOG=rpc=trace - SECONDS=0 - nice -19 cargo run --quiet --release --bin rpc -- --backoff 3000 --max-retries 100 --rpc-url $RPC --rpc-type native --jumpdest-src client-fetched-structlogs --timeout 600 fetch --start-block $BLOCK --end-block $BLOCK 1> $WITNESS - TOTALTIME=`echo -n $(($TOTALTIME + $SECONDS))` - DURATION_RPC=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` - TXALL=`grep '"jumpdest_table":' $WITNESS | wc -l` - TXNONE=`grep '"jumpdest_table": null' $WITNESS | wc -l` - TXOK=`echo -n $(($TXALL - $TXNONE))` - echo "Now testing block $BLOCK .." - export RUST_LOG=info - SECONDS=0 - timeout 10m nice -19 ./prove_stdio.sh $WITNESS test_only $BLOCK - EXITCODE=$? - TOTALTIME=`echo -n $(($TOTALTIME + $SECONDS))` - DURATION_PRV=`date -u -d @"$SECONDS" +'%-Hh%-Mm%-Ss'` - TOTALTIME=`date -u -d @"$TOTALTIME" +'%-Hh%-Mm%-Ss'` - if [ $EXITCODE -eq 0 ] - then - VERDICT="success" - else - VERDICT="failure" - fi - printf "%s %10i %s %3i %8s %8s %8s %3i %3i %3i \n" $GITHASH $BLOCK $VERDICT $EXITCODE $DURATION_RPC $DURATION_PRV $TOTALTIME $TXOK $TXNONE $TXALL | tee -a witnesses/native_results.txt - - - ### Clean up - TEST_OUT_PATH="${REPO_ROOT}/$BLOCK.test.out" - rm $TEST_OUT_PATH - rm $WITNESS - -done - From 7263ea3ae65f4b542cd1a7d9239d78e836f0b63f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 12:36:03 +0200 Subject: [PATCH 079/112] remove modifications --- .cargo/config.toml | 13 +--- Cargo.toml | 23 ------ scripts/prove_stdio.sh | 173 ----------------------------------------- 3 files changed, 1 insertion(+), 208 deletions(-) delete mode 100755 scripts/prove_stdio.sh diff --git a/.cargo/config.toml b/.cargo/config.toml index 526c1c9b4..6340ce34a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,15 +1,4 @@ [build] # https://github.com/rust-lang/rust/pull/124129 # https://github.com/dtolnay/linkme/pull/88 - -[env] -RUST_BACKTRACE = "1" -RUST_TEST_NOCAPTURE = "1" - -[term] -verbose = true -color = 'auto' - -[target.x86_64-unknown-linux-gnu] -linker = "clang" -rustflags = ["-Z", "linker-features=-lld", "-C", "target-cpu=native"] #, "-C", "link-arg=-fuse-ld=/usr/bin/mold", "-C", "debuginfo=2"] +rustflags = ["-Z", "linker-features=-lld"] diff --git a/Cargo.toml b/Cargo.toml index 5fb943015..39f4f405c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,26 +125,3 @@ starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" -[profile.release] -opt-level = 3 -debug = true -incremental = true -debug-assertions = true -lto = false -overflow-checks = false - -[profile.test] -opt-level = 3 -debug = true -incremental = true -debug-assertions = true -lto = false -overflow-checks = false - -[profile.dev] -opt-level = 3 -debug = true -incremental = true -debug-assertions = true -lto = false -overflow-checks = false diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh deleted file mode 100755 index 1e432c8f7..000000000 --- a/scripts/prove_stdio.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/bash -# ------------------------------------------------------------------------------ -set -x - -# Run prover with the parsed input from the standard terminal. -# To generate the json input file, use the `rpc` tool, for example: -# `cargo run --bin rpc -- fetch --rpc-url http://127.0.0.1:8546 --start-block 2 --end-block 5 > witness.json` - -# Args: -# 1 --> Input witness json file -# 2 --> Test run only flag `test_only` (optional) - -# We're going to set the parallelism in line with the total cpu count -if [[ "$OSTYPE" == "darwin"* ]]; then - num_procs=$(sysctl -n hw.physicalcpu) -else - num_procs=$(nproc) -fi - -# Force the working directory to always be the `tools/` directory. -REPO_ROOT=$(git rev-parse --show-toplevel) -PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs" - -BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-1}" -echo "Block batch size: $BLOCK_BATCH_SIZE" - -BATCH_SIZE=${BATCH_SIZE:-1} -echo "Batch size: $BATCH_SIZE" - -OUTPUT_LOG="${REPO_ROOT}/output.log" -PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" -TEST_OUT_PATH="${REPO_ROOT}/$3.test.out" - -# Configured Rayon and Tokio with rough defaults -export RAYON_NUM_THREADS=$num_procs -export TOKIO_WORKER_THREADS=$num_procs - -#export RUST_MIN_STACK=33554432 -export RUST_BACKTRACE=full -export RUST_LOG=info -# Script users are running locally, and might benefit from extra perf. -# See also .cargo/config.toml. -export RUSTFLAGS='-C target-cpu=native -Z linker-features=-lld' - -INPUT_FILE=$1 -TEST_ONLY=$2 - -if [[ $INPUT_FILE == "" ]]; then - echo "Please provide witness json input file, e.g. artifacts/witness_b19240705.json" - exit 1 -fi - -if [[ ! -s $INPUT_FILE ]]; then - echo "Input file $INPUT_FILE does not exist or has length 0." - exit 6 -fi - -# Circuit sizes only matter in non test_only mode. -if ! [[ $TEST_ONLY == "test_only" ]]; then - if [[ $INPUT_FILE == *"witness_b19807080"* ]]; then - # These sizes are configured specifically for block 19807080. Don't use this in other scenarios - echo "Using specific circuit sizes for witness_b19807080.json" - export ARITHMETIC_CIRCUIT_SIZE="16..18" - export BYTE_PACKING_CIRCUIT_SIZE="8..15" - export CPU_CIRCUIT_SIZE="9..20" - export KECCAK_CIRCUIT_SIZE="7..18" - export KECCAK_SPONGE_CIRCUIT_SIZE="8..14" - export LOGIC_CIRCUIT_SIZE="5..17" - export MEMORY_CIRCUIT_SIZE="17..22" - export MEMORY_BEFORE_CIRCUIT_SIZE="16..20" - export MEMORY_AFTER_CIRCUIT_SIZE="7..20" - # TODO(Robin): update Poseidon ranges here and below once Kernel ASM supports Poseidon ops - export POSEIDON_CIRCUIT_SIZE="4..8" - elif [[ $INPUT_FILE == *"witness_b3_b6"* ]]; then - # These sizes are configured specifically for custom blocks 3 to 6. Don't use this in other scenarios - echo "Using specific circuit sizes for witness_b3_b6.json" - export ARITHMETIC_CIRCUIT_SIZE="16..18" - export BYTE_PACKING_CIRCUIT_SIZE="8..15" - export CPU_CIRCUIT_SIZE="10..20" - export KECCAK_CIRCUIT_SIZE="4..13" - export KECCAK_SPONGE_CIRCUIT_SIZE="8..9" - export LOGIC_CIRCUIT_SIZE="4..14" - export MEMORY_CIRCUIT_SIZE="17..22" - export MEMORY_BEFORE_CIRCUIT_SIZE="16..18" - export MEMORY_AFTER_CIRCUIT_SIZE="7..8" - export POSEIDON_CIRCUIT_SIZE="4..8" - else - export ARITHMETIC_CIRCUIT_SIZE="16..21" - export BYTE_PACKING_CIRCUIT_SIZE="8..21" - export CPU_CIRCUIT_SIZE="8..21" - export KECCAK_CIRCUIT_SIZE="4..20" - export KECCAK_SPONGE_CIRCUIT_SIZE="8..17" - export LOGIC_CIRCUIT_SIZE="4..21" - export MEMORY_CIRCUIT_SIZE="17..24" - export MEMORY_BEFORE_CIRCUIT_SIZE="16..23" - export MEMORY_AFTER_CIRCUIT_SIZE="7..23" - export POSEIDON_CIRCUIT_SIZE="4..8" - fi -fi - - -# If we run ./prove_stdio.sh test_only, we'll generate a dummy -# proof. This is useful for quickly testing decoding and all of the -# other non-proving code. -if [[ $TEST_ONLY == "test_only" ]]; then - nice -19 cargo run --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR --batch-size $BATCH_SIZE --save-inputs-on-error stdio < $INPUT_FILE |& tee &> $TEST_OUT_PATH - if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then - echo -e "\n\nSuccess - Note this was just a test, not a proof" - #rm $TEST_OUT_PATH - exit 0 - elif grep -q 'Attempted to collapse an extension node' $TEST_OUT_PATH; then - echo "ERROR: Attempted to collapse an extension node. See $TEST_OUT_PATH for more details." - rm $TEST_OUT_PATH - exit 4 - elif grep -q 'SIMW == RPCW ? false' $TEST_OUT_PATH; then - echo "ERROR: SIMW == RPCW ? false. See $TEST_OUT_PATH for more details." - exit 5 - elif grep -q 'Proving task finished with error' $TEST_OUT_PATH; then - # Some error occurred, display the logs and exit. - echo "ERROR: Proving task finished with error. See $TEST_OUT_PATH for more details." - exit 1 - else - echo -e "\n\nUndecided. Proving process has stopped but verdict is undecided. See $TEST_OUT_PATH for more details." - exit 2 - fi -fi - -cargo build --release --jobs "$num_procs" - - -start_time=$(date +%s%N) -nice -19 "${REPO_ROOT}/target/release/leader" --runtime in-memory --load-strategy on-demand -n 1 --block-batch-size $BLOCK_BATCH_SIZE \ - --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE |& tee $OUTPUT_LOG -end_time=$(date +%s%N) - -cat $OUTPUT_LOG | grep "Successfully wrote to disk proof file " | awk '{print $NF}' | tee $PROOFS_FILE_LIST -if [ ! -s "$PROOFS_FILE_LIST" ]; then - # Some error occurred, display the logs and exit. - cat $OUTPUT_LOG - echo "Proof list not generated, some error happened. For more details check the log file $OUTPUT_LOG" - exit 1 -fi - -cat $PROOFS_FILE_LIST | while read proof_file; -do - echo "Verifying proof file $proof_file" - verify_file=$PROOF_OUTPUT_DIR/verify_$(basename $proof_file).out - nice -19 "${REPO_ROOT}/target/release/verifier" -f $proof_file | tee $verify_file - if grep -q 'All proofs verified successfully!' $verify_file; then - echo "Proof verification for file $proof_file successful"; - rm $verify_file # we keep the generated proof for potential reuse - else - # Some error occurred with verification, display the logs and exit. - cat $verify_file - echo "There was an issue with proof verification. See $verify_file for more details."; - exit 1 - fi -done - -duration_ns=$((end_time - start_time)) -duration_sec=$(echo "$duration_ns / 1000000000" | bc -l) - -echo "Success!" -echo "Proving duration:" $duration_sec " seconds" -echo "Note, this duration is inclusive of circuit handling and overall process initialization"; - -# Clean up in case of success -rm $OUTPUT_LOG - - - - - From ced5d5f9da7e0dba9eda8565987fd1426a7a5854 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 12:36:23 +0200 Subject: [PATCH 080/112] rename a --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 31857ac61..780ef5716 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -97,7 +97,7 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let clock = interpreter.get_clock(); - let (a, jdtw) = interpreter + let (jdtp, jdtw) = interpreter .generation_state .get_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); @@ -105,8 +105,8 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - interpreter.generation_state.jumpdest_table = Some(a.clone()); - Some((a, jdtw)) + interpreter.generation_state.jumpdest_table = Some(jdtp.clone()); + Some((jdtp, jdtw)) } } } From 7dc2f51653253d23eff1ef58cd79e6a60dd8c396 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 12:37:02 +0200 Subject: [PATCH 081/112] remove todo --- evm_arithmetization/src/generation/prover_input.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index b5018395a..6db978073 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -362,9 +362,6 @@ impl GenerationState { fn run_next_jumpdest_table_address(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; - // TODO(einar-polygon) - // get_code from self.memory - if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; } From 2848ede5bbe2fc3d08747da330f3195b9eeeda2f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 12:37:47 +0200 Subject: [PATCH 082/112] add derive_more and add docs --- Cargo.lock | 1 + evm_arithmetization/Cargo.toml | 1 + .../src/cpu/kernel/interpreter.rs | 24 +++-- .../src/generation/jumpdest.rs | 97 ++++++++++--------- 4 files changed, 65 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7dc061ad..7ac076699 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2025,6 +2025,7 @@ dependencies = [ "anyhow", "bytes", "criterion", + "derive_more", "env_logger 0.11.5", "ethereum-types", "hashbrown", diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index f5dfec2f2..d131d5bc2 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -17,6 +17,7 @@ keywords.workspace = true [dependencies] anyhow.workspace = true bytes.workspace = true +derive_more = "1.0.0" env_logger.workspace = true ethereum-types.workspace = true hashbrown.workspace = true diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 780ef5716..36a2034ee 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -21,7 +21,7 @@ use crate::cpu::columns::CpuColumnsView; use crate::cpu::kernel::aggregator::KERNEL; use crate::cpu::kernel::constants::global_metadata::GlobalMetadata; use crate::generation::debug_inputs; -use crate::generation::jumpdest::{ContextJumpDests, JumpDestTableProcessed, JumpDestTableWitness}; +use crate::generation::jumpdest::{Context, JumpDestTableProcessed, JumpDestTableWitness}; use crate::generation::linked_list::LinkedListsPtrs; use crate::generation::mpt::{load_linked_lists_and_txn_and_receipt_mpts, TrieRootPtrs}; use crate::generation::prover_input::get_proofs_and_jumpdests; @@ -161,6 +161,10 @@ pub(crate) fn set_registers_and_run( /// /// - `jumpdest_table_rpc`: The raw table received from RPC. /// - `code_db`: The corresponding database of contract code used in the trace. +/// +/// # Output +/// +/// Returns a [`JumpDestTableProccessed`]. pub(crate) fn get_jumpdest_analysis_inputs_rpc( jumpdest_table_rpc: &JumpDestTableWitness, code_map: &HashMap>, @@ -190,15 +194,15 @@ pub(crate) fn get_jumpdest_analysis_inputs_rpc( /// /// # Arguments /// -/// - `ctx_jumpdests`: Map from `ctx` to its list of offsets to reached -/// `JUMPDEST`s. -/// - `code`: The bytecode for the contexts. This is the same for all contexts. -fn prove_context_jumpdests( - code: &[u8], - ctx_jumpdests: &ContextJumpDests, -) -> HashMap> { - ctx_jumpdests - .0 +/// - `code`: The bytecode for the context `ctx`. +/// - `ctx`: Map from `ctx` to its list of `JUMPDEST` offsets. +/// +/// # Outputs +/// +/// Returns a [`HashMap`] from `ctx` to [`Vec`] of proofs. Each proofs ia a +/// pair. +fn prove_context_jumpdests(code: &[u8], ctx: &Context) -> HashMap> { + ctx.0 .iter() .map(|(&ctx, jumpdests)| { let proofs = jumpdests.last().map_or(Vec::default(), |&largest_address| { diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 6def65e77..55f401c46 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -26,9 +26,9 @@ use std::cmp::max; use std::{ collections::{BTreeSet, HashMap}, fmt::Display, - ops::{Deref, DerefMut}, }; +use derive_more::derive::{Deref, DerefMut}; use itertools::{sorted, Itertools}; use keccak_hash::H256; use serde::{Deserialize, Serialize}; @@ -36,18 +36,18 @@ use serde::{Deserialize, Serialize}; /// Each `CodeHash` can be called one or more times, /// each time creating a new `Context`. /// Each `Context` will contain one or more offsets of `JUMPDEST`. -#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] -pub struct ContextJumpDests(pub HashMap>); +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default, Deref, DerefMut)] +pub struct Context(pub HashMap>); /// The result after proving a [`JumpDestTableWitness`]. -#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default, Deref, DerefMut)] pub(crate) struct JumpDestTableProcessed(HashMap>); /// Map `CodeHash -> (Context -> [JumpDests])` -#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] -pub struct JumpDestTableWitness(HashMap); +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default, Deref, DerefMut)] +pub struct JumpDestTableWitness(HashMap); -impl ContextJumpDests { +impl Context { pub fn insert(&mut self, ctx: usize, offset: usize) { self.entry(ctx).or_default().insert(offset); } @@ -64,7 +64,7 @@ impl JumpDestTableProcessed { } impl JumpDestTableWitness { - pub fn get(&self, code_hash: &H256) -> Option<&ContextJumpDests> { + pub fn get(&self, code_hash: &H256) -> Option<&Context> { self.0.get(code_hash) } @@ -97,6 +97,7 @@ impl JumpDestTableWitness { } } +// The following Display instances are added to make it easier to read diffs. impl Display for JumpDestTableWitness { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { writeln!(f, "\n=== JumpDestTableWitness ===")?; @@ -108,7 +109,7 @@ impl Display for JumpDestTableWitness { } } -impl Display for ContextJumpDests { +impl Display for Context { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let v: Vec<_> = self.0.iter().sorted().collect(); for (ctx, offsets) in v.into_iter() { @@ -134,45 +135,13 @@ impl Display for JumpDestTableProcessed { } } -impl Deref for ContextJumpDests { - type Target = HashMap>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for ContextJumpDests { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl Deref for JumpDestTableProcessed { - type Target = HashMap>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for JumpDestTableProcessed { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl Deref for JumpDestTableWitness { - type Target = HashMap; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for JumpDestTableWitness { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 +impl FromIterator<(H256, usize, usize)> for JumpDestTableWitness { + fn from_iter>(iter: T) -> Self { + let mut jdtw = JumpDestTableWitness::default(); + for (code_hash, ctx, offset) in iter.into_iter() { + jdtw.insert(code_hash, ctx, offset); + } + jdtw } } @@ -209,4 +178,36 @@ mod test { assert_eq!(86, max_ctx); assert_eq!(expected, actual) } + + #[test] + fn test_extend_from_iter() { + let code_hash = H256::default(); + + let ctx_map = vec![ + (code_hash, 1, 1), + (code_hash, 2, 2), + (code_hash, 42, 3), + (code_hash, 43, 4), + ]; + let table1 = JumpDestTableWitness::from_iter(ctx_map); + let table2 = table1.clone(); + + let jdts = [&table1, &table2]; + let (actual, max_ctx) = JumpDestTableWitness::merge(jdts); + + let ctx_map_merged = vec![ + (code_hash, 1, 1), + (code_hash, 2, 2), + (code_hash, 42, 3), + (code_hash, 43, 4), + (code_hash, 44, 1), + (code_hash, 45, 2), + (code_hash, 85, 3), + (code_hash, 86, 4), + ]; + let expected = JumpDestTableWitness::from_iter(ctx_map_merged); + + assert_eq!(86, max_ctx); + assert_eq!(expected, actual) + } } From 552d5693bdf35d036647dee8e4858bcb78195c64 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 13:21:48 +0200 Subject: [PATCH 083/112] clean up --- Cargo.toml | 1 - .../src/generation/jumpdest.rs | 28 ------------------- 2 files changed, 29 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 39f4f405c..66050ed4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,4 +124,3 @@ starky = { git = "https://github.com/0xPolygonZero/plonky2.git", rev = "dc77c77f [workspace.lints.clippy] too_long_first_doc_paragraph = "allow" - diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 55f401c46..5f516e52b 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -151,34 +151,6 @@ mod test { use super::JumpDestTableWitness; - #[test] - fn test_extend() { - let code_hash = H256::default(); - - let mut table1 = JumpDestTableWitness::default(); - table1.insert(code_hash, 1, 1); - table1.insert(code_hash, 2, 2); - table1.insert(code_hash, 42, 3); - table1.insert(code_hash, 43, 4); - let table2 = table1.clone(); - - let jdts = [&table1, &table2]; - let (actual, max_ctx) = JumpDestTableWitness::merge(jdts); - - let mut expected = JumpDestTableWitness::default(); - expected.insert(code_hash, 1, 1); - expected.insert(code_hash, 2, 2); - expected.insert(code_hash, 42, 3); - expected.insert(code_hash, 43, 4); - expected.insert(code_hash, 44, 1); - expected.insert(code_hash, 45, 2); - expected.insert(code_hash, 85, 3); - expected.insert(code_hash, 86, 4); - - assert_eq!(86, max_ctx); - assert_eq!(expected, actual) - } - #[test] fn test_extend_from_iter() { let code_hash = H256::default(); From 83a08207a6c84feb83a66ea0e2ca1801855431cb Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 13:59:07 +0200 Subject: [PATCH 084/112] reinstantiate failover simulation --- .../src/generation/prover_input.rs | 34 +++++-------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 6db978073..3bc25307c 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -795,42 +795,24 @@ impl GenerationState { // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - // REVIEW: This will be rewritten to only run simulation when - // `self.inputs.jumpdest_table` is `None`. info!("Generating JUMPDEST tables"); let rpcw = self.inputs.jumpdest_table.clone(); let rpcp: Option = rpcw .as_ref() .map(|jdt| get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); - info!("Generating JUMPDEST tables: Running SIM"); + if rpcp.is_some() { + self.jumpdest_table = rpcp; + return Ok(()); + } - self.inputs.jumpdest_table = None; - let sims = simulate_cpu_and_get_user_jumps("terminate_common", self); + info!("Generating JUMPDEST tables: Running SIM"); - let (simp, ref simw): (Option, Option) = - sims.map_or_else(|| (None, None), |(sim, simw)| (Some(sim), Some(simw))); + let (simp, _simw) = simulate_cpu_and_get_user_jumps("terminate_common", self) + .ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; + self.jumpdest_table = Some(simp); info!("Generating JUMPDEST tables: finished"); - if rpcw.is_some() && simw != &rpcw { - if let Some(s) = simw { - info!("SIMW {}", s); - } - if let Some(r) = rpcw.as_ref() { - info!("RPCW {}", r); - } - info!("SIMW == RPCW ? {}", simw == &rpcw); - info!("tx: {:?}", self.inputs.txn_hashes); - // panic!(); - // info!("SIMP {:?}", &simp); - // info!("RPCP {:?}", &rpcp); - // info!("SIMP == RPCP ? {}", &simp == &rpcp); - } else { - info!("JUMPDEST tables are equal."); - } - - self.jumpdest_table = if rpcp.is_some() { rpcp } else { simp }; - Ok(()) } From 81f847c095dbd49ba735a1cf3631c5b63a336820 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 14:42:10 +0200 Subject: [PATCH 085/112] use Hash2code --- trace_decoder/src/core.rs | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 6445dbb29..d7b5e19c0 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -143,7 +143,7 @@ pub fn entrypoint( checkpoint_state_trie_root, checkpoint_consolidated_hash, contract_code: { - let initcodes = + let init_codes = byte_code .iter() .filter_map(|nonempty_txn_bytes| -> Option> { @@ -154,31 +154,11 @@ pub fn entrypoint( TxKind::Call(_address) => None, } }); - - // TODO convert to Hash2Code - let initmap: HashMap<_, _> = initcodes - .into_iter() - .map(|it| (keccak_hash::keccak(&it), it)) - .collect(); - log::trace!("Initmap {:?}", initmap); - - let contractmap: HashMap<_, _> = contract_code - .into_iter() - .map(|it| (keccak_hash::keccak(&it), it)) - .collect(); - log::trace!("Contractmap {:?}", contractmap); - - let codemap: HashMap<_, _> = code_db - .clone() - .into_iter() - .map(|it| (keccak_hash::keccak(&it), it)) - .collect(); - log::trace!("Codemap {:?}", codemap); - - let mut res = codemap; - res.extend(contractmap); - res.extend(initmap); - res + let mut result = Hash2Code::default(); + result.extend(init_codes); + result.extend(contract_code); + result.extend(code_db.clone()); + result.into_hashmap() }, block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), @@ -863,6 +843,7 @@ fn map_receipt_bytes(bytes: Vec) -> anyhow::Result> { /// trace. /// If there are any txns that create contracts, then they will also /// get added here as we process the deltas. +#[derive(Default)] struct Hash2Code { /// Key must always be [`hash`] of value. inner: HashMap>, @@ -886,6 +867,9 @@ impl Hash2Code { pub fn insert(&mut self, code: Vec) { self.inner.insert(keccak_hash::keccak(&code), code); } + pub fn into_hashmap(self) -> HashMap> { + self.inner + } } impl Extend> for Hash2Code { From 2313337f098f43362fe12b00f0e2c95aa2ca5638 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 14:49:46 +0200 Subject: [PATCH 086/112] re-add prove_stdio.sh --- scripts/prove_stdio.sh | 156 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100755 scripts/prove_stdio.sh diff --git a/scripts/prove_stdio.sh b/scripts/prove_stdio.sh new file mode 100755 index 000000000..c3c794086 --- /dev/null +++ b/scripts/prove_stdio.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# ------------------------------------------------------------------------------ +set -exo pipefail + +# Run prover with the parsed input from the standard terminal. +# To generate the json input file, use the `rpc` tool, for example: +# `cargo run --bin rpc -- fetch --rpc-url http://127.0.0.1:8546 --start-block 2 --end-block 5 > witness.json` + +# Args: +# 1 --> Input witness json file +# 2 --> Test run only flag `test_only` (optional) + +# We're going to set the parallelism in line with the total cpu count +if [[ "$OSTYPE" == "darwin"* ]]; then + num_procs=$(sysctl -n hw.physicalcpu) +else + num_procs=$(nproc) +fi + +# Force the working directory to always be the `tools/` directory. +REPO_ROOT=$(git rev-parse --show-toplevel) +PROOF_OUTPUT_DIR="${REPO_ROOT}/proofs" + +BLOCK_BATCH_SIZE="${BLOCK_BATCH_SIZE:-8}" +echo "Block batch size: $BLOCK_BATCH_SIZE" + +OUTPUT_LOG="${REPO_ROOT}/output.log" +PROOFS_FILE_LIST="${PROOF_OUTPUT_DIR}/proof_files.json" +TEST_OUT_PATH="${REPO_ROOT}/test.out" + +# Configured Rayon and Tokio with rough defaults +export RAYON_NUM_THREADS=$num_procs +export TOKIO_WORKER_THREADS=$num_procs + +export RUST_MIN_STACK=33554432 +export RUST_BACKTRACE=full +export RUST_LOG=info +# Script users are running locally, and might benefit from extra perf. +# See also .cargo/config.toml. +export RUSTFLAGS='-C target-cpu=native -Zlinker-features=-lld' + +INPUT_FILE=$1 +TEST_ONLY=$2 + +if [[ $INPUT_FILE == "" ]]; then + echo "Please provide witness json input file, e.g. artifacts/witness_b19240705.json" + exit 1 +fi + +# Circuit sizes only matter in non test_only mode. +if ! [[ $TEST_ONLY == "test_only" ]]; then + if [[ $INPUT_FILE == *"witness_b19807080"* ]]; then + # These sizes are configured specifically for block 19807080. Don't use this in other scenarios + echo "Using specific circuit sizes for witness_b19807080.json" + export ARITHMETIC_CIRCUIT_SIZE="16..18" + export BYTE_PACKING_CIRCUIT_SIZE="8..15" + export CPU_CIRCUIT_SIZE="9..20" + export KECCAK_CIRCUIT_SIZE="7..18" + export KECCAK_SPONGE_CIRCUIT_SIZE="8..14" + export LOGIC_CIRCUIT_SIZE="5..17" + export MEMORY_CIRCUIT_SIZE="17..22" + export MEMORY_BEFORE_CIRCUIT_SIZE="16..20" + export MEMORY_AFTER_CIRCUIT_SIZE="7..20" + # TODO(Robin): update Poseidon ranges here and below once Kernel ASM supports Poseidon ops + export POSEIDON_CIRCUIT_SIZE="4..8" + elif [[ $INPUT_FILE == *"witness_b3_b6"* ]]; then + # These sizes are configured specifically for custom blocks 3 to 6. Don't use this in other scenarios + echo "Using specific circuit sizes for witness_b3_b6.json" + export ARITHMETIC_CIRCUIT_SIZE="16..18" + export BYTE_PACKING_CIRCUIT_SIZE="8..15" + export CPU_CIRCUIT_SIZE="10..20" + export KECCAK_CIRCUIT_SIZE="4..13" + export KECCAK_SPONGE_CIRCUIT_SIZE="8..9" + export LOGIC_CIRCUIT_SIZE="4..14" + export MEMORY_CIRCUIT_SIZE="17..22" + export MEMORY_BEFORE_CIRCUIT_SIZE="16..18" + export MEMORY_AFTER_CIRCUIT_SIZE="7..8" + export POSEIDON_CIRCUIT_SIZE="4..8" + else + export ARITHMETIC_CIRCUIT_SIZE="16..21" + export BYTE_PACKING_CIRCUIT_SIZE="8..21" + export CPU_CIRCUIT_SIZE="8..21" + export KECCAK_CIRCUIT_SIZE="4..20" + export KECCAK_SPONGE_CIRCUIT_SIZE="8..17" + export LOGIC_CIRCUIT_SIZE="4..21" + export MEMORY_CIRCUIT_SIZE="17..24" + export MEMORY_BEFORE_CIRCUIT_SIZE="16..23" + export MEMORY_AFTER_CIRCUIT_SIZE="7..23" + export POSEIDON_CIRCUIT_SIZE="4..8" + fi +fi + + +# If we run ./prove_stdio.sh test_only, we'll generate a dummy +# proof. This is useful for quickly testing decoding and all of the +# other non-proving code. +if [[ $TEST_ONLY == "test_only" ]]; then + cargo run --quiet --release --package zero --bin leader -- --test-only --runtime in-memory --load-strategy on-demand --block-batch-size $BLOCK_BATCH_SIZE --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $TEST_OUT_PATH + if grep -q 'All proof witnesses have been generated successfully.' $TEST_OUT_PATH; then + echo -e "\n\nSuccess - Note this was just a test, not a proof" + rm $TEST_OUT_PATH + exit + else + # Some error occurred, display the logs and exit. + cat $OUT_LOG_PATH + echo "Failed to create proof witnesses. See $OUT_LOG_PATH for more details." + exit 1 + fi +fi + +cargo build --release --jobs "$num_procs" + + +start_time=$(date +%s%N) +"${REPO_ROOT}/target/release/leader" --runtime in-memory --load-strategy on-demand -n 1 --block-batch-size $BLOCK_BATCH_SIZE \ + --proof-output-dir $PROOF_OUTPUT_DIR stdio < $INPUT_FILE &> $OUTPUT_LOG +end_time=$(date +%s%N) + +cat $OUTPUT_LOG | grep "Successfully wrote to disk proof file " | awk '{print $NF}' | tee $PROOFS_FILE_LIST +if [ ! -s "$PROOFS_FILE_LIST" ]; then + # Some error occurred, display the logs and exit. + cat $OUTPUT_LOG + echo "Proof list not generated, some error happened. For more details check the log file $OUTPUT_LOG" + exit 1 +fi + +cat $PROOFS_FILE_LIST | while read proof_file; +do + echo "Verifying proof file $proof_file" + verify_file=$PROOF_OUTPUT_DIR/verify_$(basename $proof_file).out + "${REPO_ROOT}/target/release/verifier" -f $proof_file | tee $verify_file + if grep -q 'All proofs verified successfully!' $verify_file; then + echo "Proof verification for file $proof_file successful"; + rm $verify_file # we keep the generated proof for potential reuse + else + # Some error occurred with verification, display the logs and exit. + cat $verify_file + echo "There was an issue with proof verification. See $verify_file for more details."; + exit 1 + fi +done + +duration_ns=$((end_time - start_time)) +duration_sec=$(echo "$duration_ns / 1000000000" | bc -l) + +echo "Success!" +echo "Proving duration:" $duration_sec " seconds" +echo "Note, this duration is inclusive of circuit handling and overall process initialization"; + +# Clean up in case of success +rm $OUTPUT_LOG + + + + + From 206e9a468782e47333984047fdfe26bd659ee1d2 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 14:54:05 +0200 Subject: [PATCH 087/112] mv derive_more --- Cargo.toml | 1 + evm_arithmetization/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 66050ed4b..175fbbebd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ ciborium-io = "0.2.2" clap = { version = "4.5.7", features = ["derive", "env"] } compat = { path = "compat" } criterion = "0.5.1" +derive_more = "1.0.0" dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" diff --git a/evm_arithmetization/Cargo.toml b/evm_arithmetization/Cargo.toml index d131d5bc2..9749eb084 100644 --- a/evm_arithmetization/Cargo.toml +++ b/evm_arithmetization/Cargo.toml @@ -17,7 +17,7 @@ keywords.workspace = true [dependencies] anyhow.workspace = true bytes.workspace = true -derive_more = "1.0.0" +derive_more.workspace = true env_logger.workspace = true ethereum-types.workspace = true hashbrown.workspace = true From 0b7e99732bebb9a4036bc27bb057796c864d067b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 14:57:53 +0200 Subject: [PATCH 088/112] cleanup --- zero/src/rpc/jumpdest.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 250513c82..7f323551a 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -204,9 +204,6 @@ pub(crate) fn generate_jumpdest_table<'a>( }; let callee_address = stack_value_to_address(address); - let callee_address_old = stack_value_to_address_old(address); - assert_eq!(callee_address, callee_address_old); - if callee_addr_to_code_hash.contains_key(&callee_address) { let next_code_hash = callee_addr_to_code_hash[&callee_address]; call_stack.push((next_code_hash, next_ctx_available)); @@ -333,14 +330,6 @@ fn stack_value_to_address(operand: &Uint<256, 4>) -> Address { Address::from(lower_20_bytes) } -// Review: to be removed -fn stack_value_to_address_old(operand: &Uint<256, 4>) -> Address { - let all_bytes: [u8; 32] = operand.compat().into(); - let aaa: H256 = all_bytes.into(); - let lower_20_bytes: [u8; 20] = __compat_primitive_types::H160::from(aaa).into(); - Address::from(lower_20_bytes) -} - fn trace_to_tx_structlog(tx_hash: Option, trace: GethTrace) -> Option { match trace { GethTrace::Default(structlog_frame) => { From d67911e5a21cf7751fd874287c11f27953238c1b Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 14:47:17 +0200 Subject: [PATCH 089/112] remove tracing --- .../src/cpu/kernel/interpreter.rs | 9 +-- .../src/generation/prover_input.rs | 14 +---- trace_decoder/src/core.rs | 4 -- zero/src/rpc/jumpdest.rs | 61 +------------------ 4 files changed, 5 insertions(+), 83 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 36a2034ee..3d5d46220 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -11,7 +11,7 @@ use std::collections::{BTreeSet, HashMap}; use anyhow::anyhow; use ethereum_types::{BigEndianHash, U256}; use keccak_hash::H256; -use log::{trace, Level}; +use log::Level; use mpt_trie::partial_trie::PartialTrie; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -177,13 +177,6 @@ pub(crate) fn get_jumpdest_analysis_inputs_rpc( } else { &vec![] }; - trace!( - "code: {:?}, code_addr: {:?}, {:?} <============", - &code, - &code_addr, - code_map.contains_key(code_addr), - ); - trace!("code_map: {:?}", &code_map); prove_context_jumpdests(code, ctx_jumpdests) }) .collect(); diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 3bc25307c..995b22b23 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -7,7 +7,6 @@ use anyhow::{bail, Error, Result}; use ethereum_types::{BigEndianHash, H256, U256, U512}; use itertools::Itertools; use keccak_hash::keccak; -use log::{info, trace}; use num_bigint::BigUint; use plonky2::hash::hash_types::RichField; use serde::{Deserialize, Serialize}; @@ -792,10 +791,6 @@ impl GenerationState { /// Simulate the user's code and store all the jump addresses with their /// respective contexts. fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { - // Simulate the user's code and (unnecessarily) part of the kernel code, - // skipping the validate table call - - info!("Generating JUMPDEST tables"); let rpcw = self.inputs.jumpdest_table.clone(); let rpcp: Option = rpcw .as_ref() @@ -804,15 +799,11 @@ impl GenerationState { self.jumpdest_table = rpcp; return Ok(()); } - - info!("Generating JUMPDEST tables: Running SIM"); - + // Simulate the user's code and (unnecessarily) part of the kernel code, + // skipping the validate table call let (simp, _simw) = simulate_cpu_and_get_user_jumps("terminate_common", self) .ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; self.jumpdest_table = Some(simp); - - info!("Generating JUMPDEST tables: finished"); - Ok(()) } @@ -828,7 +819,6 @@ impl GenerationState { |(ctx, jumpdest_table)| { let code = self.get_code(ctx).unwrap(); let code_hash = keccak(code.clone()); - trace!("ctx: {ctx}, code_hash: {:?} code: {:?}", code_hash, code); for offset in jumpdest_table.clone() { jdtw.insert(code_hash, ctx, offset); } diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index d7b5e19c0..97969bf52 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -21,7 +21,6 @@ use evm_arithmetization::{ }; use itertools::Itertools as _; use keccak_hash::H256; -use log::debug; use mpt_trie::partial_trie::PartialTrie as _; use nunny::NonEmpty; use zk_evm_common::gwei_to_wei; @@ -859,9 +858,6 @@ impl Hash2Code { } pub fn get(&mut self, hash: H256) -> Option> { let res = self.inner.get(&hash).cloned(); - if res.is_none() { - debug!("no code for hash {:#x}", hash); - } res } pub fn insert(&mut self, code: Vec) { diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 7f323551a..4f41b79d7 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -24,7 +24,7 @@ use tokio::time::timeout; use trace_decoder::is_precompile; use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; -use tracing::{trace, warn}; +use tracing::warn; use crate::rpc::H256; @@ -70,7 +70,6 @@ where match timeout(*fetch_timeout, block_stackonly_structlog_traces_fut).await { Ok(traces) => traces?, Err(elapsed) => { - trace!(target: "fetching block structlogs timed out", ?elapsed); bail!(elapsed); } }; @@ -97,8 +96,6 @@ pub(crate) fn generate_jumpdest_table<'a>( structlog: &[StructLog], tx_traces: impl Iterator, ) -> anyhow::Result { - trace!("Generating JUMPDEST table for tx: {}", tx.hash); - let mut jumpdest_table = JumpDestTableWitness::default(); // This map does neither contain the `init` field of Contract Deployment @@ -112,17 +109,6 @@ pub(crate) fn generate_jumpdest_table<'a>( }) .collect(); - // REVIEW: will be removed before merge - trace!( - "Transaction: {} is a {}.", - tx.hash, - if tx.to.is_some() { - "message call" - } else { - "contract creation" - } - ); - let entrypoint_code_hash: H256 = match tx.to { Some(to_address) if is_precompile(to_address.compat()) => return Ok(jumpdest_table), Some(to_address) if callee_addr_to_code_hash.contains_key(&to_address).not() => { @@ -147,7 +133,7 @@ pub(crate) fn generate_jumpdest_table<'a>( next_ctx_available += 1; let mut stuctlog_iter = structlog.iter().enumerate().peekable(); - while let Some((step, entry)) = stuctlog_iter.next() { + while let Some((_step, entry)) = stuctlog_iter.next() { let op = entry.op.as_str(); let curr_depth: usize = entry.depth.try_into().unwrap(); @@ -163,22 +149,6 @@ pub(crate) fn generate_jumpdest_table<'a>( ); let (code_hash, ctx) = call_stack.last().unwrap(); - // REVIEW: will be removed before merge - trace!( - step, - curr_depth, - tx_hash = ?tx.hash, - ?code_hash, - ctx, - next_ctx_available, - pc = entry.pc, - pc_hex = format!("{:08x?}", entry.pc), - gas = entry.gas, - gas_cost = entry.gas_cost, - op, - ?entry, - ); - match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { prev_jump = None; @@ -191,7 +161,6 @@ pub(crate) fn generate_jumpdest_table<'a>( let operands_used = 2; if evm_stack.len() < operands_used { - trace!( "Opcode {op} expected {operands_used} operands at the EVM stack, but only {} were found.", evm_stack.len()); // Note for future debugging: There may exist edge cases, where the call // context has been incremented before the call op fails. This should be // accounted for before this and the following `continue`. The details are @@ -209,21 +178,6 @@ pub(crate) fn generate_jumpdest_table<'a>( call_stack.push((next_code_hash, next_ctx_available)); }; - if is_precompile(callee_address.compat()) { - trace!("Called precompile at address {}.", &callee_address); - }; - - if callee_addr_to_code_hash.contains_key(&callee_address).not() - && is_precompile(callee_address.compat()).not() - { - // This case happens if calling an EOA. This is described - // under opcode `STOP`: https://www.evm.codes/#00?fork=cancun - trace!( - "Callee address {} has no associated `code_hash`.", - &callee_address - ); - } - if let Some((_next_step, next_entry)) = stuctlog_iter.peek() { let next_depth: usize = next_entry.depth.try_into().unwrap(); if next_depth < curr_depth { @@ -252,7 +206,6 @@ pub(crate) fn generate_jumpdest_table<'a>( let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); let operands = 1; if evm_stack.len() < operands { - trace!( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len() ); continue; } let [jump_target, ..] = evm_stack[..] else { @@ -268,7 +221,6 @@ pub(crate) fn generate_jumpdest_table<'a>( let evm_stack: Vec<_> = entry.stack.as_ref().unwrap().iter().rev().collect(); let operands = 2; if evm_stack.len() < operands { - trace!( "Opcode {op} expected {operands} operands at the EVM stack, but only {} were found.", evm_stack.len()); continue; }; @@ -290,20 +242,11 @@ pub(crate) fn generate_jumpdest_table<'a>( prev_jump = None; if jumped_here.not() { - trace!( - "{op}: JUMPDESTs at offset {} was reached through fall-through.", - entry.pc - ); continue; } let jumpdest_offset = TryInto::::try_into(entry.pc); if jumpdest_offset.is_err() { - trace!( - "{op}: Could not cast offset {} to usize {}.", - entry.pc, - usize::MAX - ); continue; } ensure!(jumpdest_offset.unwrap() < 24576); From 6b88ff6d64dc7adcf1715b6497a705279ef6f1b9 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 21 Oct 2024 15:58:59 +0200 Subject: [PATCH 090/112] fix derive_more --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ece36735e..a03c58399 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ ciborium-io = "0.2.2" clap = { version = "4.5.7", features = ["derive", "env"] } compat = { path = "compat" } criterion = "0.5.1" -derive_more = "1.0.0" +derive_more = { version = "1.0.0", features = ["deref", "deref_mut"] } dotenvy = "0.15.7" either = "1.12.0" enum-as-inner = "0.6.0" From b1d5eaae61998c1a0287a02d1e80d837a8a2afea Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 4 Nov 2024 19:39:17 +0100 Subject: [PATCH 091/112] . --- zero/src/rpc/jerigon.rs | 2 +- zero/src/rpc/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index 5c29fe966..9156fb937 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -63,7 +63,7 @@ where let block: Block = cached_provider .get_block(target_block_id, BlockTransactionsKind::Full) .await? - .context("No block")?; + .context("no block")?; let block_jumpdest_table_witnesses: Vec> = match jumpdest_src { JumpdestSrc::ProverSimulation => vec![None; tx_results.len()], diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index 14bb20300..ebee7bb07 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -56,7 +56,7 @@ pub async fn block_prover_input( checkpoint_block_number: u64, jumpdest_src: JumpdestSrc, fetch_timeout: Duration, -) -> anyhow::Result +) -> Result where ProviderT: Provider, TransportT: Transport + Clone, From 078d95a40df96a27c079b5cc3f47c3029e2085fd Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 5 Nov 2024 22:04:44 +0100 Subject: [PATCH 092/112] . --- .../cpu/kernel/asm/core/jumpdest_analysis.asm | 16 ++++++++-------- .../src/generation/prover_input.rs | 1 + trace_decoder/src/core.rs | 7 +++++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/asm/core/jumpdest_analysis.asm b/evm_arithmetization/src/cpu/kernel/asm/core/jumpdest_analysis.asm index 842d794a9..b99260723 100644 --- a/evm_arithmetization/src/cpu/kernel/asm/core/jumpdest_analysis.asm +++ b/evm_arithmetization/src/cpu/kernel/asm/core/jumpdest_analysis.asm @@ -1,4 +1,4 @@ -// Set @SEGMENT_JUMPDEST_BITS to one between positions [init_pos, final_pos], +// Set @SEGMENT_JUMPDEST_BITS to one between positions [init_pos, final_pos], // for the given context's code. // Pre stack: init_pos, ctx, final_pos, retdest // Post stack: (empty) @@ -16,12 +16,12 @@ loop: DUP2 DUP2 GT // i > final_pos %jumpi(proof_not_ok) - // stack: i, final_pos, retdest + // stack: i, final_pos, retdest DUP1 MLOAD_GENERAL // SEGMENT_CODE == 0 // stack: opcode, i, final_pos, retdest - DUP1 + DUP1 // Slightly more efficient than `%eq_const(0x5b) ISZERO` PUSH 0x5b SUB @@ -141,7 +141,7 @@ global write_table_if_jumpdest: // stack: proof_prefix_addr, ctx, jumpdest, retdest // If we are here we need to check that the next 32 bytes are not // PUSHXX for XX > 32 - n, n in {1, 32}. - + %stack (proof_prefix_addr, ctx) -> (ctx, proof_prefix_addr, 32, proof_prefix_addr, ctx) @@ -214,11 +214,11 @@ return: // non-deterministically guessing the sequence of jumpdest // addresses used during program execution within the current context. // For each jumpdest address we also non-deterministically guess -// a proof, which is another address in the code such that +// a proof, which is another address in the code such that // is_jumpdest doesn't abort, when the proof is at the top of the stack // an the jumpdest address below. If that's the case we set the // corresponding bit in @SEGMENT_JUMPDEST_BITS to 1. -// +// // stack: ctx, code_len, retdest // stack: (empty) global jumpdest_analysis: @@ -238,7 +238,7 @@ check_proof: // stack: address, ctx, code_len, retdest DUP3 DUP2 %assert_le %decrement - // stack: proof, ctx, code_len, retdest + // stack: address, ctx, code_len, retdest DUP2 SWAP1 // stack: address, ctx, ctx, code_len, retdest // We read the proof @@ -246,7 +246,7 @@ check_proof: // stack: proof, address, ctx, ctx, code_len, retdest %write_table_if_jumpdest // stack: ctx, code_len, retdest - + %jump(jumpdest_analysis) %macro jumpdest_analysis diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 995b22b23..8b2a62878 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -801,6 +801,7 @@ impl GenerationState { } // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call + self.jumpdest_table = None; let (simp, _simw) = simulate_cpu_and_get_user_jumps("terminate_common", self) .ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; self.jumpdest_table = Some(simp); diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 9a149dc60..47b25746e 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -210,10 +210,13 @@ pub fn entrypoint( // causing failover to simulating jumpdest analysis for // the whole batch. There is an optimization opportunity // here. - jumpdest_tables + dbg!(&jumpdest_tables); + let res = jumpdest_tables .into_iter() .collect::>>() - .map(|jdt| JumpDestTableWitness::merge(jdt.iter()).0) + .map(|jdt| JumpDestTableWitness::merge(jdt.iter()).0); + dbg!(&res); + res }, } }, From 89d37564652a5cffcfd819d04189baaabdc5b961 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 7 Nov 2024 16:11:51 +0100 Subject: [PATCH 093/112] fix: search for witness index --- .../src/generation/jumpdest.rs | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 5f516e52b..a97e25c1b 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -41,7 +41,13 @@ pub struct Context(pub HashMap>); /// The result after proving a [`JumpDestTableWitness`]. #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default, Deref, DerefMut)] -pub(crate) struct JumpDestTableProcessed(HashMap>); +pub(crate) struct JumpDestTableProcessed { + pub contexts: HashMap>, + /// Translates batch index to a wittness index + pub index: HashMap, + largest_batch_index: usize, + largest_witness_index: usize, +} /// Map `CodeHash -> (Context -> [JumpDests])` #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default, Deref, DerefMut)] @@ -59,7 +65,33 @@ impl Context { impl JumpDestTableProcessed { pub fn new(ctx_map: HashMap>) -> Self { - Self(ctx_map) + Self { + contexts: ctx_map, + // mapping from batch indices to witness indices + index: Default::default(), + largest_batch_index: 0, + largest_witness_index: 0, + } + } + + pub fn try_get_ctx_mut(&mut self, batch_ctx: &usize) -> Option<&mut Vec> { + if *batch_ctx <= self.largest_batch_index { + let witness_index = self.index[batch_ctx]; + return self.contexts.get_mut(&witness_index); + } + + let mut new_witness_index = self.largest_witness_index; + for i in new_witness_index + 1..*batch_ctx { + if self.contexts.contains_key(&i) { + new_witness_index = i; + break; + } + } + + self.largest_witness_index = new_witness_index; + self.largest_batch_index = *batch_ctx; + self.index.insert(*batch_ctx, new_witness_index); + self.contexts.get_mut(&new_witness_index) } } From 04e2df9a94854d1bc237f977577cf93db663faaf Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 7 Nov 2024 16:51:13 +0100 Subject: [PATCH 094/112] remove ctx --- .../src/cpu/kernel/tests/core/jumpdest_analysis.rs | 2 +- evm_arithmetization/src/generation/jumpdest.rs | 9 +++++++-- evm_arithmetization/src/generation/prover_input.rs | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index 0bb07eaf5..fd3a3cb7d 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -133,7 +133,7 @@ fn test_jumpdest_analysis() -> Result<()> { .jumpdest_table .as_mut() .unwrap() - .get_mut(&CONTEXT) + .try_get_ctx_mut(&CONTEXT) .unwrap() .pop(); interpreter diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index a97e25c1b..7f3384df6 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -40,7 +40,7 @@ use serde::{Deserialize, Serialize}; pub struct Context(pub HashMap>); /// The result after proving a [`JumpDestTableWitness`]. -#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default, Deref, DerefMut)] +#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct JumpDestTableProcessed { pub contexts: HashMap>, /// Translates batch index to a wittness index @@ -93,6 +93,11 @@ impl JumpDestTableProcessed { self.index.insert(*batch_ctx, new_witness_index); self.contexts.get_mut(&new_witness_index) } + + pub fn remove_ctx(&mut self, batch_ctx: &usize) { + let witness_index = self.index[batch_ctx]; + self.contexts.remove(&witness_index); + } } impl JumpDestTableWitness { @@ -159,7 +164,7 @@ impl Display for JumpDestTableProcessed { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { writeln!(f, "\n=== JumpDestTableProcessed ===")?; - let v = sorted(self.0.clone()); + let v = sorted(self.contexts.clone()); for (ctx, code) in v { writeln!(f, "ctx: {:?} {:?}", ctx, code)?; } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 8b2a62878..b2b9d1492 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -371,12 +371,12 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) + if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_ctx_mut(&context) && let Some(next_jumpdest_address) = ctx_jumpdest_table.pop() { Ok((next_jumpdest_address + 1).into()) } else { - jumpdest_table.remove(&context); + jumpdest_table.remove_ctx(&context); Ok(U256::zero()) } } @@ -390,7 +390,7 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = jumpdest_table.get_mut(&context) + if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_ctx_mut(&context) && let Some(next_jumpdest_proof) = ctx_jumpdest_table.pop() { Ok(next_jumpdest_proof.into()) From 3df0292a86945bbe3d36809c446877c959009f8e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 8 Nov 2024 00:33:41 +0100 Subject: [PATCH 095/112] workaround --- .../src/cpu/kernel/interpreter.rs | 3 +- .../src/generation/jumpdest.rs | 50 +++++++++++++++---- .../src/generation/prover_input.rs | 12 ++++- evm_arithmetization/src/generation/state.rs | 4 ++ 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index c0904d8f3..97be14562 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -170,6 +170,7 @@ pub(crate) fn set_registers_and_run( pub(crate) fn get_jumpdest_analysis_inputs_rpc( jumpdest_table_rpc: &JumpDestTableWitness, code_map: &HashMap>, + prev: usize, ) -> JumpDestTableProcessed { let ctx_proofs = (*jumpdest_table_rpc) .iter() @@ -182,7 +183,7 @@ pub(crate) fn get_jumpdest_analysis_inputs_rpc( prove_context_jumpdests(code, ctx_jumpdests) }) .collect(); - JumpDestTableProcessed::new(ctx_proofs) + JumpDestTableProcessed::new_with_start(ctx_proofs, prev) } /// Orchestrates the proving of all contexts in a specific bytecode. diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 7f3384df6..342faf1ea 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -42,9 +42,9 @@ pub struct Context(pub HashMap>); /// The result after proving a [`JumpDestTableWitness`]. #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct JumpDestTableProcessed { - pub contexts: HashMap>, + witness_contexts: HashMap>, /// Translates batch index to a wittness index - pub index: HashMap, + index: HashMap, largest_batch_index: usize, largest_witness_index: usize, } @@ -66,7 +66,7 @@ impl Context { impl JumpDestTableProcessed { pub fn new(ctx_map: HashMap>) -> Self { Self { - contexts: ctx_map, + witness_contexts: ctx_map, // mapping from batch indices to witness indices index: Default::default(), largest_batch_index: 0, @@ -74,29 +74,59 @@ impl JumpDestTableProcessed { } } + pub fn new_with_start(ctx_map: HashMap>, start_ctx: usize) -> Self { + Self { + witness_contexts: ctx_map, + // mapping from batch indices to witness indices + index: Default::default(), + largest_batch_index: 0, + largest_witness_index: start_ctx, + } + } + pub fn try_get_ctx_mut(&mut self, batch_ctx: &usize) -> Option<&mut Vec> { + log::info!( + "START: {}->{} {:#?}", + self.largest_batch_index, + self.largest_witness_index, + self.index + ); if *batch_ctx <= self.largest_batch_index { let witness_index = self.index[batch_ctx]; - return self.contexts.get_mut(&witness_index); + return self.witness_contexts.get_mut(&witness_index); } + self.largest_batch_index = *batch_ctx; let mut new_witness_index = self.largest_witness_index; - for i in new_witness_index + 1..*batch_ctx { - if self.contexts.contains_key(&i) { + for i in new_witness_index + 1..=*batch_ctx { + if self.witness_contexts.contains_key(&i) { new_witness_index = i; break; } } self.largest_witness_index = new_witness_index; - self.largest_batch_index = *batch_ctx; self.index.insert(*batch_ctx, new_witness_index); - self.contexts.get_mut(&new_witness_index) + log::info!( + "END: {}->{} {:#?}", + self.largest_batch_index, + self.largest_witness_index, + self.index + ); + self.witness_contexts.get_mut(&new_witness_index) } pub fn remove_ctx(&mut self, batch_ctx: &usize) { let witness_index = self.index[batch_ctx]; - self.contexts.remove(&witness_index); + self.witness_contexts.remove(&witness_index); + } + + pub fn last_ctx(self) -> usize { + self.witness_contexts + .keys() + .max() + .copied() + .unwrap_or_default() } } @@ -164,7 +194,7 @@ impl Display for JumpDestTableProcessed { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { writeln!(f, "\n=== JumpDestTableProcessed ===")?; - let v = sorted(self.contexts.clone()); + let v = sorted(self.witness_contexts.clone()); for (ctx, code) in v { writeln!(f, "ctx: {:?} {:?}", ctx, code)?; } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index b2b9d1492..9f81d81b2 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -361,9 +361,18 @@ impl GenerationState { fn run_next_jumpdest_table_address(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; + log::info!("CONTEXT {} NEXT {}", context, self.next_txn_index); + + let curr_idx = self.next_txn_index - 1; + if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; } + let prev = self.max_ctx.get(curr_idx - 1).copied().unwrap_or(0); + if curr_idx == self.max_ctx.len() { + self.max_ctx.push(prev + context) + } + self.max_ctx[curr_idx] = std::cmp::max(self.max_ctx[curr_idx], prev + context); let Some(jumpdest_table) = &mut self.jumpdest_table else { return Err(ProgramError::ProverInputError( @@ -791,10 +800,11 @@ impl GenerationState { /// Simulate the user's code and store all the jump addresses with their /// respective contexts. fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { + let prev = self.max_ctx.last().copied().unwrap_or(0); let rpcw = self.inputs.jumpdest_table.clone(); let rpcp: Option = rpcw .as_ref() - .map(|jdt| get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code)); + .map(|jdt| get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, prev)); if rpcp.is_some() { self.jumpdest_table = rpcp; return Ok(()); diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index 1ff7b3f7c..92bf58129 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -396,6 +396,8 @@ pub struct GenerationState { /// Provides quick access to pointers that reference the memory location of /// either and account or a slot in the respective access list. pub(crate) state_ptrs: LinkedListsPtrs, + + pub(crate) max_ctx: Vec, } impl GenerationState { @@ -461,6 +463,7 @@ impl GenerationState { access_lists_ptrs: LinkedListsPtrs::default(), state_ptrs: LinkedListsPtrs::default(), ger_prover_inputs, + max_ctx: vec![], }; let trie_root_ptrs = state.preinitialize_linked_lists_and_txn_and_receipt_mpts(&inputs.tries); @@ -576,6 +579,7 @@ impl GenerationState { jumpdest_table: None, access_lists_ptrs: self.access_lists_ptrs.clone(), state_ptrs: self.state_ptrs.clone(), + max_ctx: vec![], } } From bd02ee3b57bc52a478eff5b706ab8e3ee09e9a49 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 8 Nov 2024 16:27:47 +0100 Subject: [PATCH 096/112] . --- evm_arithmetization/src/cpu/kernel/interpreter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 97be14562..4a3842a5c 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -172,7 +172,7 @@ pub(crate) fn get_jumpdest_analysis_inputs_rpc( code_map: &HashMap>, prev: usize, ) -> JumpDestTableProcessed { - let ctx_proofs = (*jumpdest_table_rpc) + let ctx_proofs = jumpdest_table_rpc .iter() .flat_map(|(code_addr, ctx_jumpdests)| { let code = if code_map.contains_key(code_addr) { From 7b7cb46061d2351008bedca4508d5ffd2bae1676 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 11 Nov 2024 12:15:31 +0100 Subject: [PATCH 097/112] bug investigations --- .../cpu/kernel/asm/core/jumpdest_analysis.asm | 5 +- .../src/cpu/kernel/interpreter.rs | 28 +++++--- .../kernel/tests/core/jumpdest_analysis.rs | 2 +- .../src/generation/jumpdest.rs | 72 +++++++++++-------- .../src/generation/prover_input.rs | 64 +++++++++++------ .../src/generation/segments.rs | 1 + evm_arithmetization/src/generation/state.rs | 12 ++-- 7 files changed, 114 insertions(+), 70 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/asm/core/jumpdest_analysis.asm b/evm_arithmetization/src/cpu/kernel/asm/core/jumpdest_analysis.asm index b99260723..7da6ad924 100644 --- a/evm_arithmetization/src/cpu/kernel/asm/core/jumpdest_analysis.asm +++ b/evm_arithmetization/src/cpu/kernel/asm/core/jumpdest_analysis.asm @@ -5,10 +5,10 @@ global verify_path_and_write_jumpdest_table: SWAP2 DUP2 - ADD // final_addr + ADD // final_addr = final_pos + ctx, i = init_pos // stack: final_addr, ctx, i, retdest SWAP2 - ADD // init_addr + ADD // init_addr = i + ctx loop: // stack: i, final_pos, retdest DUP2 DUP2 EQ // i == final_pos @@ -235,6 +235,7 @@ global jumpdest_analysis_end: %pop2 JUMP check_proof: + // stack: address + 1, ctx, code_len, retdest // stack: address, ctx, code_len, retdest DUP3 DUP2 %assert_le %decrement diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 4a3842a5c..0ba28834c 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -59,7 +59,8 @@ pub(crate) struct Interpreter { /// halt_context pub(crate) halt_context: Option, /// A table of call contexts and the JUMPDEST offsets that they jumped to. - jumpdest_table: HashMap>, + // todo + jumpdest_table_interpreter: HashMap>, /// `true` if the we are currently carrying out a jumpdest analysis. pub(crate) is_jumpdest_analysis: bool, /// Holds the value of the clock: the clock counts the number of operations @@ -80,8 +81,7 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( state: &GenerationState, ) -> Option<(JumpDestTableProcessed, JumpDestTableWitness)> { match state.jumpdest_table { - Some(_) => Default::default(), - None => { + _ => { let halt_pc = KERNEL.global_labels[final_label]; let initial_context = state.registers.context; let mut interpreter = Interpreter::new_with_state_and_halt_condition( @@ -95,19 +95,22 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( let _ = interpreter.run(); - log::trace!("jumpdest table = {:?}", interpreter.jumpdest_table); + log::trace!( + "jumpdest table = {:?}", + interpreter.jumpdest_table_interpreter + ); let clock = interpreter.get_clock(); let (jdtp, jdtw) = interpreter .generation_state - .get_jumpdest_analysis_inputs(interpreter.jumpdest_table.clone()); + .get_jumpdest_analysis_inputs(interpreter.jumpdest_table_interpreter.clone()); log::debug!( "Simulated CPU for jumpdest analysis halted after {:?} cycles.", clock ); - interpreter.generation_state.jumpdest_table = Some(jdtp.clone()); + // interpreter.generation_state.jumpdest_table = Some(jdtp.clone()); Some((jdtp, jdtw)) } } @@ -121,6 +124,7 @@ pub(crate) struct ExtraSegmentData { pub(crate) withdrawal_prover_inputs: Vec, pub(crate) ger_prover_inputs: Vec, pub(crate) trie_root_ptrs: TrieRootPtrs, + // todo pub(crate) jumpdest_table: Option, pub(crate) access_lists_ptrs: LinkedListsPtrs, pub(crate) state_ptrs: LinkedListsPtrs, @@ -178,6 +182,7 @@ pub(crate) fn get_jumpdest_analysis_inputs_rpc( let code = if code_map.contains_key(code_addr) { &code_map[code_addr] } else { + panic!("code not found"); &vec![] }; prove_context_jumpdests(code, ctx_jumpdests) @@ -239,7 +244,8 @@ impl Interpreter { halt_context: None, #[cfg(test)] opcode_count: HashMap::new(), - jumpdest_table: HashMap::new(), + // todo + jumpdest_table_interpreter: HashMap::new(), is_jumpdest_analysis: false, clock: 0, max_cpu_len_log, @@ -271,7 +277,8 @@ impl Interpreter { halt_context: Some(halt_context), #[cfg(test)] opcode_count: HashMap::new(), - jumpdest_table: HashMap::new(), + // check + jumpdest_table_interpreter: HashMap::new(), is_jumpdest_analysis: true, clock: 0, max_cpu_len_log, @@ -530,14 +537,15 @@ impl Interpreter { .content } + // what happens here? pub(crate) fn add_jumpdest_offset(&mut self, offset: usize) { if let Some(jumpdest_table) = self - .jumpdest_table + .jumpdest_table_interpreter .get_mut(&self.generation_state.registers.context) { jumpdest_table.insert(offset); } else { - self.jumpdest_table.insert( + self.jumpdest_table_interpreter.insert( self.generation_state.registers.context, BTreeSet::from([offset]), ); diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index fd3a3cb7d..a00ca27b4 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -103,7 +103,7 @@ fn test_jumpdest_analysis() -> Result<()> { ), )])); - // The `set_jumpdest_analysis_inputs` method is never used. + // TODO The `set_jumpdest_analysis_inputs` method is never used. assert_eq!( interpreter.generation_state.jumpdest_table, // Context 3 has jumpdest 1, 5, 7. All have proof 0 and hence diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 342faf1ea..7c1301cb9 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -23,6 +23,7 @@ //! [`JUMPDEST`]: https://www.evm.codes/?fork=cancun#5b use std::cmp::max; +use std::ops::Not as _; use std::{ collections::{BTreeSet, HashMap}, fmt::Display, @@ -45,8 +46,8 @@ pub(crate) struct JumpDestTableProcessed { witness_contexts: HashMap>, /// Translates batch index to a wittness index index: HashMap, - largest_batch_index: usize, - largest_witness_index: usize, + largest_batch_ctx: usize, + pub largest_witness_ctx: usize, } /// Map `CodeHash -> (Context -> [JumpDests])` @@ -69,8 +70,8 @@ impl JumpDestTableProcessed { witness_contexts: ctx_map, // mapping from batch indices to witness indices index: Default::default(), - largest_batch_index: 0, - largest_witness_index: 0, + largest_batch_ctx: 0, + largest_witness_ctx: 0, } } @@ -79,41 +80,45 @@ impl JumpDestTableProcessed { witness_contexts: ctx_map, // mapping from batch indices to witness indices index: Default::default(), - largest_batch_index: 0, - largest_witness_index: start_ctx, + largest_batch_ctx: 0, + largest_witness_ctx: start_ctx, } } pub fn try_get_ctx_mut(&mut self, batch_ctx: &usize) -> Option<&mut Vec> { log::info!( - "START: {}->{} {:#?}", - self.largest_batch_index, - self.largest_witness_index, + "START: batch_ctx {} :: max_b {} :: max-w {} {:#?}", + batch_ctx, + self.largest_batch_ctx, + self.largest_witness_ctx, self.index ); - if *batch_ctx <= self.largest_batch_index { - let witness_index = self.index[batch_ctx]; - return self.witness_contexts.get_mut(&witness_index); + + if *batch_ctx <= self.largest_batch_ctx { + let witness_ctx = self.index[batch_ctx]; + return self.witness_contexts.get_mut(&witness_ctx); } - self.largest_batch_index = *batch_ctx; + self.largest_batch_ctx = *batch_ctx; - let mut new_witness_index = self.largest_witness_index; - for i in new_witness_index + 1..=*batch_ctx { + let mut new_witness_ctx = self.largest_witness_ctx; + for i in (self.largest_witness_ctx + 1).. { if self.witness_contexts.contains_key(&i) { - new_witness_index = i; + new_witness_ctx = i; break; } } - self.largest_witness_index = new_witness_index; - self.index.insert(*batch_ctx, new_witness_index); + self.largest_witness_ctx = new_witness_ctx; + self.index.insert(*batch_ctx, new_witness_ctx); log::info!( - "END: {}->{} {:#?}", - self.largest_batch_index, - self.largest_witness_index, + "END:{} {}->{} {:#?}", + batch_ctx, + self.largest_batch_ctx, + self.largest_witness_ctx, self.index ); - self.witness_contexts.get_mut(&new_witness_index) + + self.witness_contexts.get_mut(&new_witness_ctx) } pub fn remove_ctx(&mut self, batch_ctx: &usize) { @@ -121,13 +126,22 @@ impl JumpDestTableProcessed { self.witness_contexts.remove(&witness_index); } - pub fn last_ctx(self) -> usize { - self.witness_contexts - .keys() - .max() - .copied() - .unwrap_or_default() - } + // pub fn last_ctx(self) -> usize { + // self.witness_contexts + // .keys() + // .max() + // .copied() + // .unwrap_or_default() + // } + + // pub fn is_subset(&self, other: &Self) -> bool { + // for (k, v) in self.witness_contexts.iter() { + // if other.witness_contexts.contains_key(k).not() || v != + // &other.witness_contexts[k] { return false; + // } + // } + // true + // } } impl JumpDestTableWitness { diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 9f81d81b2..70c898d2e 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -86,7 +86,7 @@ impl GenerationState { fn run_end_of_txns(&mut self) -> Result { // Reset the jumpdest table before the next transaction. - self.jumpdest_table = None; + // self.jumpdest_table = None; let end = self.next_txn_index == self.inputs.txn_hashes.len(); if end { Ok(U256::one()) @@ -357,22 +357,17 @@ impl GenerationState { .ok_or(ProgramError::ProverInputError(OutOfGerData)) } - /// Returns the next used jump address. + /// Returns the next used jumpdest address. fn run_next_jumpdest_table_address(&mut self) -> Result { - let context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; + let batch_context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; - log::info!("CONTEXT {} NEXT {}", context, self.next_txn_index); + log::info!("CONTEXT {} NEXT {}", batch_context, self.next_txn_index); - let curr_idx = self.next_txn_index - 1; + let curr_txn_idx = self.next_txn_index - 1; if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; } - let prev = self.max_ctx.get(curr_idx - 1).copied().unwrap_or(0); - if curr_idx == self.max_ctx.len() { - self.max_ctx.push(prev + context) - } - self.max_ctx[curr_idx] = std::cmp::max(self.max_ctx[curr_idx], prev + context); let Some(jumpdest_table) = &mut self.jumpdest_table else { return Err(ProgramError::ProverInputError( @@ -380,12 +375,23 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_ctx_mut(&context) + if [2, 3, 4].contains(&batch_context) { + return Ok(U256::zero()); + } + + if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_ctx_mut(&batch_context) && let Some(next_jumpdest_address) = ctx_jumpdest_table.pop() { + if curr_txn_idx + 1 != self.max_wctx.len() { + self.max_wctx.push(0) + } + self.max_wctx[curr_txn_idx] = std::cmp::max( + self.max_wctx[curr_txn_idx], + jumpdest_table.largest_witness_ctx, + ); Ok((next_jumpdest_address + 1).into()) } else { - jumpdest_table.remove_ctx(&context); + jumpdest_table.remove_ctx(&batch_context); Ok(U256::zero()) } } @@ -800,22 +806,34 @@ impl GenerationState { /// Simulate the user's code and store all the jump addresses with their /// respective contexts. fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { - let prev = self.max_ctx.last().copied().unwrap_or(0); + let prev_max_wctx = self.max_wctx.last().copied().unwrap_or(0); let rpcw = self.inputs.jumpdest_table.clone(); - let rpcp: Option = rpcw - .as_ref() - .map(|jdt| get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, prev)); - if rpcp.is_some() { - self.jumpdest_table = rpcp; - return Ok(()); - } + log::info!("{:#?}", &rpcw); + let rpcp: Option = rpcw.as_ref().map(|jdt| { + get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, prev_max_wctx) + }); + // if rpcp.is_some() { + // self.jumpdest_table = rpcp; + // return Ok(()); + // } // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call self.jumpdest_table = None; - let (simp, _simw) = simulate_cpu_and_get_user_jumps("terminate_common", self) + let (simp, simw) = simulate_cpu_and_get_user_jumps("terminate_common", &*self) .ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; - self.jumpdest_table = Some(simp); - Ok(()) + // self.jumpdest_table = Some(simp.clone()); + log::info!("{:#?}", &rpcw); + log::info!("{:#?}", &rpcp); + log::info!("{:#?}", &simw); + log::info!("{:#?}", &simp); + + // if rpcp.is_some() { + // dbg!(rpcp.as_ref(), Some(&simp)); + // // assert!(simp.is_subset(rpcp.as_ref().unwrap())); + // self.jumpdest_table = rpcp; + // } + self.jumpdest_table = rpcp; + return Ok(()); } /// Given a HashMap containing the contexts and the jumpdest addresses, diff --git a/evm_arithmetization/src/generation/segments.rs b/evm_arithmetization/src/generation/segments.rs index 1df63af29..f5e6a85a5 100644 --- a/evm_arithmetization/src/generation/segments.rs +++ b/evm_arithmetization/src/generation/segments.rs @@ -81,6 +81,7 @@ fn build_segment_data( .clone(), ger_prover_inputs: interpreter.generation_state.ger_prover_inputs.clone(), trie_root_ptrs: interpreter.generation_state.trie_root_ptrs.clone(), + // todo verify jumpdest_table: interpreter.generation_state.jumpdest_table.clone(), next_txn_index: interpreter.generation_state.next_txn_index, access_lists_ptrs: interpreter.generation_state.access_lists_ptrs.clone(), diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index 92bf58129..d54c4b411 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -101,6 +101,7 @@ pub(crate) trait State { } /// Returns the context in which the jumpdest analysis should end. + // this seems pointless fn get_halt_context(&self) -> Option { None } @@ -387,8 +388,8 @@ pub struct GenerationState { /// "proof" for a jump destination is either 0 or an address i > 32 in /// the code (not necessarily pointing to an opcode) such that for every /// j in [i, i+32] it holds that code[j] < 0x7f - j + i. + // jumpdest_table: Option, pub(crate) jumpdest_table: Option, - /// Provides quick access to pointers that reference the location /// of either and account or a slot in the respective access list. pub(crate) access_lists_ptrs: LinkedListsPtrs, @@ -397,7 +398,7 @@ pub struct GenerationState { /// either and account or a slot in the respective access list. pub(crate) state_ptrs: LinkedListsPtrs, - pub(crate) max_ctx: Vec, + pub(crate) max_wctx: Vec, } impl GenerationState { @@ -463,7 +464,7 @@ impl GenerationState { access_lists_ptrs: LinkedListsPtrs::default(), state_ptrs: LinkedListsPtrs::default(), ger_prover_inputs, - max_ctx: vec![], + max_wctx: vec![], }; let trie_root_ptrs = state.preinitialize_linked_lists_and_txn_and_receipt_mpts(&inputs.tries); @@ -576,10 +577,10 @@ impl GenerationState { txn_root_ptr: 0, receipt_root_ptr: 0, }, - jumpdest_table: None, + jumpdest_table: self.jumpdest_table.clone(), access_lists_ptrs: self.access_lists_ptrs.clone(), state_ptrs: self.state_ptrs.clone(), - max_ctx: vec![], + max_wctx: self.max_wctx.clone(), } } @@ -594,6 +595,7 @@ impl GenerationState { .clone_from(&segment_data.extra_data.ger_prover_inputs); self.trie_root_ptrs .clone_from(&segment_data.extra_data.trie_root_ptrs); + // todo verify self.jumpdest_table .clone_from(&segment_data.extra_data.jumpdest_table); self.state_ptrs From 8811d23607e31ced5c85db97195a9c49f63e9e47 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 11 Nov 2024 18:05:14 +0100 Subject: [PATCH 098/112] before going back --- .../benches/fibonacci_25m_gas.rs | 2 +- .../src/cpu/kernel/tests/add11.rs | 4 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 2 +- .../src/generation/jumpdest.rs | 77 +++++++++++-------- evm_arithmetization/src/generation/mod.rs | 4 +- .../src/generation/prover_input.rs | 61 ++++++++++----- evm_arithmetization/src/generation/state.rs | 1 + evm_arithmetization/tests/add11_yml.rs | 3 +- evm_arithmetization/tests/erc20.rs | 2 +- evm_arithmetization/tests/erc721.rs | 2 +- evm_arithmetization/tests/log_opcode.rs | 2 +- evm_arithmetization/tests/selfdestruct.rs | 2 +- evm_arithmetization/tests/simple_transfer.rs | 2 +- evm_arithmetization/tests/withdrawals.rs | 2 +- trace_decoder/src/core.rs | 18 +++-- 15 files changed, 113 insertions(+), 71 deletions(-) diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index 38344629c..aa810efb7 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -192,7 +192,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: None, + jumpdest_table: vec![None], }) } diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index 71933ec3e..ae7042389 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -193,7 +193,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: None, + jumpdest_table: vec![None], }; let initial_stack = vec![]; @@ -371,7 +371,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: None, + jumpdest_table: vec![None], }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index 27374888c..0046238ba 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,7 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, - jumpdest_table: None, + jumpdest_table: vec![None], }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 7c1301cb9..0ed1d3bf3 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -86,44 +86,53 @@ impl JumpDestTableProcessed { } pub fn try_get_ctx_mut(&mut self, batch_ctx: &usize) -> Option<&mut Vec> { - log::info!( - "START: batch_ctx {} :: max_b {} :: max-w {} {:#?}", - batch_ctx, - self.largest_batch_ctx, - self.largest_witness_ctx, - self.index - ); - - if *batch_ctx <= self.largest_batch_ctx { - let witness_ctx = self.index[batch_ctx]; - return self.witness_contexts.get_mut(&witness_ctx); - } - self.largest_batch_ctx = *batch_ctx; + log::info!("query_ctx {}", batch_ctx,); + self.witness_contexts.get_mut(batch_ctx) + } - let mut new_witness_ctx = self.largest_witness_ctx; - for i in (self.largest_witness_ctx + 1).. { - if self.witness_contexts.contains_key(&i) { - new_witness_ctx = i; - break; - } - } + // pub fn try_get_ctx_mut(&mut self, batch_ctx: &usize) -> Option<&mut + // Vec> { log::info!( + // "START: batch_ctx {} :: max_b {} :: max-w {} {:#?}", + // batch_ctx, + // self.largest_batch_ctx, + // self.largest_witness_ctx, + // self.index + // ); + + // if *batch_ctx <= self.largest_batch_ctx { + // let witness_ctx = self.index[batch_ctx]; + // return self.witness_contexts.get_mut(&witness_ctx); + // } + // self.largest_batch_ctx = *batch_ctx; - self.largest_witness_ctx = new_witness_ctx; - self.index.insert(*batch_ctx, new_witness_ctx); - log::info!( - "END:{} {}->{} {:#?}", - batch_ctx, - self.largest_batch_ctx, - self.largest_witness_ctx, - self.index - ); - - self.witness_contexts.get_mut(&new_witness_ctx) - } + // let mut new_witness_ctx = self.largest_witness_ctx; + // for i in (self.largest_witness_ctx + 1).. { + // if self.witness_contexts.contains_key(&i) { + // new_witness_ctx = i; + // break; + // } + // } + + // self.largest_witness_ctx = new_witness_ctx; + // self.index.insert(*batch_ctx, new_witness_ctx); + // log::info!( + // "END:{} {}->{} {:#?}", + // batch_ctx, + // self.largest_batch_ctx, + // self.largest_witness_ctx, + // self.index + // ); + + // self.witness_contexts.get_mut(&new_witness_ctx) + // } + + // pub fn remove_ctx(&mut self, batch_ctx: &usize) { + // let witness_index = self.index[batch_ctx]; + // self.witness_contexts.remove(&witness_index); + // } pub fn remove_ctx(&mut self, batch_ctx: &usize) { - let witness_index = self.index[batch_ctx]; - self.witness_contexts.remove(&witness_index); + self.witness_contexts.remove(&batch_ctx); } // pub fn last_ctx(self) -> usize { diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index 66253e4a4..d35098570 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -136,7 +136,7 @@ pub struct GenerationInputs { /// A table listing each JUMPDESTs reached in each call context under /// associated code hash. - pub jumpdest_table: Option, + pub jumpdest_table: Vec>, } /// A lighter version of [`GenerationInputs`], which have been trimmed @@ -190,7 +190,7 @@ pub struct TrimmedGenerationInputs { /// A list of tables listing each JUMPDESTs reached in each call context /// under associated code hash. - pub jumpdest_table: Option, + pub jumpdest_table: Vec>, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 70c898d2e..283adc189 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -375,20 +375,20 @@ impl GenerationState { )); }; - if [2, 3, 4].contains(&batch_context) { - return Ok(U256::zero()); - } + // if [2, 3, 4].contains(&batch_context) { + // return Ok(U256::zero()); + // } if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_ctx_mut(&batch_context) && let Some(next_jumpdest_address) = ctx_jumpdest_table.pop() { - if curr_txn_idx + 1 != self.max_wctx.len() { - self.max_wctx.push(0) - } - self.max_wctx[curr_txn_idx] = std::cmp::max( - self.max_wctx[curr_txn_idx], - jumpdest_table.largest_witness_ctx, - ); + // if curr_txn_idx + 1 != self.max_wctx.len() { + // self.max_wctx.push(0) + // } + // self.max_wctx[curr_txn_idx] = std::cmp::max( + // self.max_wctx[curr_txn_idx], + // jumpdest_table.largest_witness_ctx, + // ); Ok((next_jumpdest_address + 1).into()) } else { jumpdest_table.remove_ctx(&batch_context); @@ -806,12 +806,36 @@ impl GenerationState { /// Simulate the user's code and store all the jump addresses with their /// respective contexts. fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { - let prev_max_wctx = self.max_wctx.last().copied().unwrap_or(0); - let rpcw = self.inputs.jumpdest_table.clone(); + // let prev_max_wctx = self.max_wctx.last().copied().unwrap_or(0); + let prev_max_wctx: usize = 0; + // self + // .inputs + // .jumpdest_table + // .get(self.next_txn_index - 1) + // .map(|x| x.as_ref()) + // .flatten() + // .map(|jdt| { + // jdt.iter() + // .map(|(_h, jdt)| jdt.keys().max().copied().unwrap_or(0)) + // .max() + // .unwrap_or(0) + // }) + // .unwrap_or(0) + // + 4; + let tx_batch_order = self.next_txn_index; + log::info!("TXNUM: {}", self.next_txn_index); + log::info!("TXLEN: {}", self.inputs.txn_hashes.len()); + log::info!("TX: {}", self.inputs.txn_number_before); + let idx = if 0 < self.next_txn_index { + self.next_txn_index - 1 + } else { + 0 + }; + let rpcw = self.inputs.jumpdest_table[idx].clone(); log::info!("{:#?}", &rpcw); - let rpcp: Option = rpcw.as_ref().map(|jdt| { - get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, prev_max_wctx) - }); + // let rpcp: Option = rpcw.as_ref().map(|jdt| { + // get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, + // prev_max_wctx) }); // if rpcp.is_some() { // self.jumpdest_table = rpcp; // return Ok(()); @@ -823,16 +847,17 @@ impl GenerationState { .ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; // self.jumpdest_table = Some(simp.clone()); log::info!("{:#?}", &rpcw); - log::info!("{:#?}", &rpcp); + // log::info!("{:#?}", &rpcp); log::info!("{:#?}", &simw); - log::info!("{:#?}", &simp); + // log::info!("{:#?}", &simp); // if rpcp.is_some() { // dbg!(rpcp.as_ref(), Some(&simp)); // // assert!(simp.is_subset(rpcp.as_ref().unwrap())); // self.jumpdest_table = rpcp; // } - self.jumpdest_table = rpcp; + // self.jumpdest_table = rpcp; + self.jumpdest_table = Some(simp); return Ok(()); } diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index d54c4b411..9d77a4968 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -355,6 +355,7 @@ pub struct GenerationState { pub(crate) memory: MemoryState, pub(crate) traces: Traces, + /// In the batch / block?? pub(crate) next_txn_index: usize, /// Memory used by stale contexts can be pruned so proving segments can be diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 87d959925..5e7d44fca 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,7 +200,8 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: None, + jumpdest_table: vec![None], + } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index 2cb9a538b..f27f4738d 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -195,7 +195,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: None, + jumpdest_table: vec![None], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index f34d5c621..df16e869c 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -199,7 +199,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: None, + jumpdest_table: vec![None], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index 871fa90f7..14534def5 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -266,7 +266,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: None, + jumpdest_table: vec![None], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index eaf56dea6..7cfe1a750 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -170,7 +170,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: None, + jumpdest_table: vec![None], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index f40eadc67..9896ea772 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -162,7 +162,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: None, + jumpdest_table: vec![None], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index 4b3656cb6..2f976fd68 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -105,7 +105,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: None, + jumpdest_table: vec![None], }; let max_cpu_len_log = 20; diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 47b25746e..ce9d04272 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -2,6 +2,7 @@ use core::{convert::Into as _, option::Option::None}; use std::{ cmp, collections::{BTreeMap, BTreeSet, HashMap}, + iter::repeat, mem, }; @@ -211,12 +212,17 @@ pub fn entrypoint( // the whole batch. There is an optimization opportunity // here. dbg!(&jumpdest_tables); - let res = jumpdest_tables - .into_iter() - .collect::>>() - .map(|jdt| JumpDestTableWitness::merge(jdt.iter()).0); - dbg!(&res); - res + // let res = jumpdest_tables + // .into_iter() + // .collect::>>() + // .map(|jdt| JumpDestTableWitness::merge(jdt.iter()).0); + // dbg!(&res); + + if jumpdest_tables.iter().any(Option::is_none) { + repeat(None).take(jumpdest_tables.len()).collect::>() + } else { + jumpdest_tables + } }, } }, From c9bc69dc2ebc38974185279aca2890e3fee232a2 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 11 Nov 2024 20:40:38 +0100 Subject: [PATCH 099/112] . --- .../src/generation/prover_input.rs | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 283adc189..78e98346f 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -363,8 +363,6 @@ impl GenerationState { log::info!("CONTEXT {} NEXT {}", batch_context, self.next_txn_index); - let curr_txn_idx = self.next_txn_index - 1; - if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; } @@ -830,16 +828,16 @@ impl GenerationState { self.next_txn_index - 1 } else { 0 - }; + }; let rpcw = self.inputs.jumpdest_table[idx].clone(); log::info!("{:#?}", &rpcw); - // let rpcp: Option = rpcw.as_ref().map(|jdt| { - // get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, - // prev_max_wctx) }); - // if rpcp.is_some() { - // self.jumpdest_table = rpcp; - // return Ok(()); - // } + let rpcp: Option = rpcw.as_ref().map(|jdt| { + get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, prev_max_wctx) + }); + if rpcp.is_some() { + self.jumpdest_table = rpcp; + return Ok(()); + } // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call self.jumpdest_table = None; @@ -851,11 +849,11 @@ impl GenerationState { log::info!("{:#?}", &simw); // log::info!("{:#?}", &simp); - // if rpcp.is_some() { - // dbg!(rpcp.as_ref(), Some(&simp)); - // // assert!(simp.is_subset(rpcp.as_ref().unwrap())); - // self.jumpdest_table = rpcp; - // } + if rpcp.is_some() { + dbg!(rpcp.as_ref(), Some(&simp)); + // assert!(simp.is_subset(rpcp.as_ref().unwrap())); + self.jumpdest_table = rpcp; + } // self.jumpdest_table = rpcp; self.jumpdest_table = Some(simp); return Ok(()); From 200f6ac30f5f358b9967a07bcacee54af9ad4d66 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Mon, 11 Nov 2024 22:10:39 +0100 Subject: [PATCH 100/112] remove max_wctx --- evm_arithmetization/src/generation/prover_input.rs | 11 ----------- evm_arithmetization/src/generation/state.rs | 4 ---- 2 files changed, 15 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 78e98346f..fd8f59086 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -373,20 +373,9 @@ impl GenerationState { )); }; - // if [2, 3, 4].contains(&batch_context) { - // return Ok(U256::zero()); - // } - if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_ctx_mut(&batch_context) && let Some(next_jumpdest_address) = ctx_jumpdest_table.pop() { - // if curr_txn_idx + 1 != self.max_wctx.len() { - // self.max_wctx.push(0) - // } - // self.max_wctx[curr_txn_idx] = std::cmp::max( - // self.max_wctx[curr_txn_idx], - // jumpdest_table.largest_witness_ctx, - // ); Ok((next_jumpdest_address + 1).into()) } else { jumpdest_table.remove_ctx(&batch_context); diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index 9d77a4968..77aeb2400 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -398,8 +398,6 @@ pub struct GenerationState { /// Provides quick access to pointers that reference the memory location of /// either and account or a slot in the respective access list. pub(crate) state_ptrs: LinkedListsPtrs, - - pub(crate) max_wctx: Vec, } impl GenerationState { @@ -465,7 +463,6 @@ impl GenerationState { access_lists_ptrs: LinkedListsPtrs::default(), state_ptrs: LinkedListsPtrs::default(), ger_prover_inputs, - max_wctx: vec![], }; let trie_root_ptrs = state.preinitialize_linked_lists_and_txn_and_receipt_mpts(&inputs.tries); @@ -581,7 +578,6 @@ impl GenerationState { jumpdest_table: self.jumpdest_table.clone(), access_lists_ptrs: self.access_lists_ptrs.clone(), state_ptrs: self.state_ptrs.clone(), - max_wctx: self.max_wctx.clone(), } } From 3f283c954096182ca58e04e159e197189bb2052e Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 12 Nov 2024 14:13:04 +0100 Subject: [PATCH 101/112] . --- .../benches/fibonacci_25m_gas.rs | 2 +- .../src/cpu/kernel/tests/add11.rs | 4 +- .../src/cpu/kernel/tests/init_exc_stop.rs | 2 +- evm_arithmetization/src/generation/mod.rs | 4 +- .../src/generation/prover_input.rs | 63 +++++++++---------- evm_arithmetization/tests/add11_yml.rs | 3 +- evm_arithmetization/tests/erc20.rs | 2 +- evm_arithmetization/tests/erc721.rs | 2 +- evm_arithmetization/tests/log_opcode.rs | 2 +- evm_arithmetization/tests/selfdestruct.rs | 2 +- evm_arithmetization/tests/simple_transfer.rs | 2 +- evm_arithmetization/tests/withdrawals.rs | 2 +- trace_decoder/src/core.rs | 2 +- 13 files changed, 44 insertions(+), 48 deletions(-) diff --git a/evm_arithmetization/benches/fibonacci_25m_gas.rs b/evm_arithmetization/benches/fibonacci_25m_gas.rs index aa810efb7..8b752919e 100644 --- a/evm_arithmetization/benches/fibonacci_25m_gas.rs +++ b/evm_arithmetization/benches/fibonacci_25m_gas.rs @@ -192,7 +192,7 @@ fn prepare_setup() -> anyhow::Result> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: vec![None], + batch_jumpdest_tables: vec![None], }) } diff --git a/evm_arithmetization/src/cpu/kernel/tests/add11.rs b/evm_arithmetization/src/cpu/kernel/tests/add11.rs index ae7042389..e46886585 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/add11.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/add11.rs @@ -193,7 +193,7 @@ fn test_add11_yml() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: vec![None], + batch_jumpdest_tables: vec![None], }; let initial_stack = vec![]; @@ -371,7 +371,7 @@ fn test_add11_yml_with_exception() { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: vec![None], + batch_jumpdest_tables: vec![None], }; let initial_stack = vec![]; diff --git a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs index 0046238ba..9000870ac 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/init_exc_stop.rs @@ -101,7 +101,7 @@ fn test_init_exc_stop() { cur_hash: H256::default(), }, ger_data: None, - jumpdest_table: vec![None], + batch_jumpdest_tables: vec![None], }; let initial_stack = vec![]; let initial_offset = KERNEL.global_labels["init"]; diff --git a/evm_arithmetization/src/generation/mod.rs b/evm_arithmetization/src/generation/mod.rs index d35098570..a5a2872ea 100644 --- a/evm_arithmetization/src/generation/mod.rs +++ b/evm_arithmetization/src/generation/mod.rs @@ -136,7 +136,7 @@ pub struct GenerationInputs { /// A table listing each JUMPDESTs reached in each call context under /// associated code hash. - pub jumpdest_table: Vec>, + pub batch_jumpdest_tables: Vec>, } /// A lighter version of [`GenerationInputs`], which have been trimmed @@ -265,7 +265,7 @@ impl GenerationInputs { burn_addr: self.burn_addr, block_metadata: self.block_metadata.clone(), block_hashes: self.block_hashes.clone(), - jumpdest_table: self.jumpdest_table.clone(), + jumpdest_table: self.batch_jumpdest_tables.clone(), } } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index fd8f59086..392ead387 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -361,7 +361,11 @@ impl GenerationState { fn run_next_jumpdest_table_address(&mut self) -> Result { let batch_context = u256_to_usize(stack_peek(self, 0)? >> CONTEXT_SCALING_FACTOR)?; - log::info!("CONTEXT {} NEXT {}", batch_context, self.next_txn_index); + log::info!( + "Current ctx {} current tx {}", + batch_context, + self.next_txn_index - 1 + ); if self.jumpdest_table.is_none() { self.generate_jumpdest_table()?; @@ -793,50 +797,43 @@ impl GenerationState { /// Simulate the user's code and store all the jump addresses with their /// respective contexts. fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { - // let prev_max_wctx = self.max_wctx.last().copied().unwrap_or(0); - let prev_max_wctx: usize = 0; - // self - // .inputs - // .jumpdest_table - // .get(self.next_txn_index - 1) - // .map(|x| x.as_ref()) - // .flatten() - // .map(|jdt| { - // jdt.iter() - // .map(|(_h, jdt)| jdt.keys().max().copied().unwrap_or(0)) - // .max() - // .unwrap_or(0) - // }) - // .unwrap_or(0) - // + 4; - let tx_batch_order = self.next_txn_index; + let tx_in_batch_idx = self.next_txn_index - 1; + let prev_max_wctx: usize = self + .inputs + .jumpdest_table + .get(tx_in_batch_idx - 1) + .map(|x| x.as_ref()) + .flatten() + .map(|jdt| { + jdt.iter() + .map(|(_h, jdt)| jdt.keys().max().copied().unwrap_or(0)) + .max() + .unwrap_or(0) + }) + .unwrap_or(0) + + 6; + log::info!("TXOFFSET: {}", tx_in_batch_idx); log::info!("TXNUM: {}", self.next_txn_index); log::info!("TXLEN: {}", self.inputs.txn_hashes.len()); log::info!("TX: {}", self.inputs.txn_number_before); - let idx = if 0 < self.next_txn_index { - self.next_txn_index - 1 - } else { - 0 - }; - let rpcw = self.inputs.jumpdest_table[idx].clone(); - log::info!("{:#?}", &rpcw); + let rpcw = self.inputs.jumpdest_table[tx_in_batch_idx].clone(); let rpcp: Option = rpcw.as_ref().map(|jdt| { get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, prev_max_wctx) }); - if rpcp.is_some() { - self.jumpdest_table = rpcp; - return Ok(()); - } + log::info!("RPCW {:#?}", &rpcw); + log::info!("RPCP {:#?}", &rpcp); + // if rpcp.is_some() { + // self.jumpdest_table = rpcp; + // return Ok(()); + // } // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call self.jumpdest_table = None; let (simp, simw) = simulate_cpu_and_get_user_jumps("terminate_common", &*self) .ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; // self.jumpdest_table = Some(simp.clone()); - log::info!("{:#?}", &rpcw); - // log::info!("{:#?}", &rpcp); - log::info!("{:#?}", &simw); - // log::info!("{:#?}", &simp); + log::info!("SIMW {:#?}", &simw); + log::info!("SIMP {:#?}", &simp); if rpcp.is_some() { dbg!(rpcp.as_ref(), Some(&simp)); diff --git a/evm_arithmetization/tests/add11_yml.rs b/evm_arithmetization/tests/add11_yml.rs index 5e7d44fca..d5ca67640 100644 --- a/evm_arithmetization/tests/add11_yml.rs +++ b/evm_arithmetization/tests/add11_yml.rs @@ -200,8 +200,7 @@ fn get_generation_inputs() -> GenerationInputs { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: vec![None], - + batch_jumpdest_tables: vec![None], } } /// The `add11_yml` test case from https://github.com/ethereum/tests diff --git a/evm_arithmetization/tests/erc20.rs b/evm_arithmetization/tests/erc20.rs index f27f4738d..36d97058a 100644 --- a/evm_arithmetization/tests/erc20.rs +++ b/evm_arithmetization/tests/erc20.rs @@ -195,7 +195,7 @@ fn test_erc20() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: vec![None], + batch_jumpdest_tables: vec![None], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/erc721.rs b/evm_arithmetization/tests/erc721.rs index df16e869c..1c0d15f76 100644 --- a/evm_arithmetization/tests/erc721.rs +++ b/evm_arithmetization/tests/erc721.rs @@ -199,7 +199,7 @@ fn test_erc721() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: vec![None], + batch_jumpdest_tables: vec![None], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/log_opcode.rs b/evm_arithmetization/tests/log_opcode.rs index 14534def5..93b666bbd 100644 --- a/evm_arithmetization/tests/log_opcode.rs +++ b/evm_arithmetization/tests/log_opcode.rs @@ -266,7 +266,7 @@ fn test_log_opcodes() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: vec![None], + batch_jumpdest_tables: vec![None], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/selfdestruct.rs b/evm_arithmetization/tests/selfdestruct.rs index 7cfe1a750..6271745ab 100644 --- a/evm_arithmetization/tests/selfdestruct.rs +++ b/evm_arithmetization/tests/selfdestruct.rs @@ -170,7 +170,7 @@ fn test_selfdestruct() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: vec![None], + batch_jumpdest_tables: vec![None], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/simple_transfer.rs b/evm_arithmetization/tests/simple_transfer.rs index 9896ea772..9347f31ca 100644 --- a/evm_arithmetization/tests/simple_transfer.rs +++ b/evm_arithmetization/tests/simple_transfer.rs @@ -162,7 +162,7 @@ fn test_simple_transfer() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: vec![None], + batch_jumpdest_tables: vec![None], }; let max_cpu_len_log = 20; diff --git a/evm_arithmetization/tests/withdrawals.rs b/evm_arithmetization/tests/withdrawals.rs index 2f976fd68..079e36e48 100644 --- a/evm_arithmetization/tests/withdrawals.rs +++ b/evm_arithmetization/tests/withdrawals.rs @@ -105,7 +105,7 @@ fn test_withdrawals() -> anyhow::Result<()> { prev_hashes: vec![H256::default(); 256], cur_hash: H256::default(), }, - jumpdest_table: vec![None], + batch_jumpdest_tables: vec![None], }; let max_cpu_len_log = 20; diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index ce9d04272..a13a4c93e 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -204,7 +204,7 @@ pub fn entrypoint( block_metadata: b_meta.clone(), block_hashes: b_hashes.clone(), burn_addr, - jumpdest_table: { + batch_jumpdest_tables: { // TODO(einar-polygon): // Note that this causes any batch containing just a // single `None` to collapse into a `None`, which From 4483db6842238f7c88abb5289d64b9f3fcd72aa6 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Tue, 12 Nov 2024 23:52:39 +0100 Subject: [PATCH 102/112] add empty contexts --- .../src/generation/jumpdest.rs | 42 +++++++++++++++---- .../src/generation/prover_input.rs | 2 +- zero/src/rpc/jumpdest.rs | 4 +- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 0ed1d3bf3..29ce6ffb2 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -55,13 +55,17 @@ pub(crate) struct JumpDestTableProcessed { pub struct JumpDestTableWitness(HashMap); impl Context { - pub fn insert(&mut self, ctx: usize, offset: usize) { - self.entry(ctx).or_default().insert(offset); - } - pub fn get(&self, ctx: usize) -> Option<&BTreeSet> { self.0.get(&ctx) } + + pub fn insert(&mut self, ctx: usize, offset_opt: Option) { + let context = self.entry(ctx).or_default(); + + if let Some(offset) = offset_opt { + context.insert(offset); + }; + } } impl JumpDestTableProcessed { @@ -160,8 +164,11 @@ impl JumpDestTableWitness { /// Insert `offset` into `ctx` under the corresponding `code_hash`. /// Creates the required `ctx` keys and `code_hash`. Idempotent. - pub fn insert(&mut self, code_hash: H256, ctx: usize, offset: usize) { - (*self).entry(code_hash).or_default().insert(ctx, offset); + pub fn insert(&mut self, code_hash: H256, ctx: usize, offset_opt: Option) { + (*self) + .entry(code_hash) + .or_default() + .insert(ctx, offset_opt); } pub fn extend(mut self, other: &Self, prev_max_ctx: usize) -> (Self, usize) { @@ -173,7 +180,7 @@ impl JumpDestTableWitness { curr_max_ctx = max(curr_max_ctx, batch_ctx); for offset in jumpdests { - self.insert(*code_hash, batch_ctx, *offset); + self.insert(*code_hash, batch_ctx, Some(*offset)); } } } @@ -229,7 +236,7 @@ impl FromIterator<(H256, usize, usize)> for JumpDestTableWitness { fn from_iter>(iter: T) -> Self { let mut jdtw = JumpDestTableWitness::default(); for (code_hash, ctx, offset) in iter.into_iter() { - jdtw.insert(code_hash, ctx, offset); + jdtw.insert(code_hash, ctx, Some(offset)); } jdtw } @@ -237,9 +244,12 @@ impl FromIterator<(H256, usize, usize)> for JumpDestTableWitness { #[cfg(test)] mod test { + use std::collections::{BTreeSet, HashMap}; + use keccak_hash::H256; use super::JumpDestTableWitness; + use crate::jumpdest::Context; #[test] fn test_extend_from_iter() { @@ -272,4 +282,20 @@ mod test { assert_eq!(86, max_ctx); assert_eq!(expected, actual) } + + #[test] + fn test_create_context() { + let code_hash = H256::default(); + let mut table1 = JumpDestTableWitness::default(); + table1.insert(code_hash, 42, None); + + let offsets = BTreeSet::::default(); + let mut ctx = HashMap::::default(); + ctx.insert(42, offsets); + let mut contexts = HashMap::::default(); + contexts.insert(code_hash, Context(ctx)); + let table2 = JumpDestTableWitness(contexts); + + assert_eq!(table1, table2); + } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 392ead387..83fb5b18d 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -858,7 +858,7 @@ impl GenerationState { let code = self.get_code(ctx).unwrap(); let code_hash = keccak(code.clone()); for offset in jumpdest_table.clone() { - jdtw.insert(code_hash, ctx, offset); + jdtw.insert(code_hash, ctx, Some(offset)); } if let Some(&largest_address) = jumpdest_table.last() { let proofs = get_proofs_and_jumpdests(&code, largest_address, jumpdest_table); diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 4f41b79d7..31d4e1f7e 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -148,6 +148,8 @@ pub(crate) fn generate_jumpdest_table<'a>( "Call stack was unexpectedly empty." ); let (code_hash, ctx) = call_stack.last().unwrap(); + tracing::info!("INSERT {} {}", *code_hash, *ctx); + jumpdest_table.insert(*code_hash, *ctx, None); match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { @@ -250,7 +252,7 @@ pub(crate) fn generate_jumpdest_table<'a>( continue; } ensure!(jumpdest_offset.unwrap() < 24576); - jumpdest_table.insert(*code_hash, *ctx, jumpdest_offset.unwrap()); + jumpdest_table.insert(*code_hash, *ctx, Some(jumpdest_offset.unwrap())); } "EXTCODECOPY" | "EXTCODESIZE" => { prev_jump = None; From 1d8b230488f6fe004b50e50adb509fdde5e6df2f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 13 Nov 2024 00:19:43 +0100 Subject: [PATCH 103/112] insert extra contexts. confirm empty empty tables --- zero/src/rpc/jumpdest.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 31d4e1f7e..57578bfad 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -24,7 +24,7 @@ use tokio::time::timeout; use trace_decoder::is_precompile; use trace_decoder::ContractCodeUsage; use trace_decoder::TxnTrace; -use tracing::warn; +use tracing::{info, warn}; use crate::rpc::H256; @@ -147,8 +147,8 @@ pub(crate) fn generate_jumpdest_table<'a>( call_stack.is_empty().not(), "Call stack was unexpectedly empty." ); - let (code_hash, ctx) = call_stack.last().unwrap(); - tracing::info!("INSERT {} {}", *code_hash, *ctx); + let (ref code_hash, ref ctx) = call_stack.last().unwrap().clone(); + info!("INSERT {} {}", *code_hash, *ctx); jumpdest_table.insert(*code_hash, *ctx, None); match op { @@ -193,6 +193,7 @@ pub(crate) fn generate_jumpdest_table<'a>( // exception or not. But this is of no consequence to the // generated Jumpdest table, so we can ignore the case. + jumpdest_table.insert(*code_hash, next_ctx_available, None); next_ctx_available += 1; } "CREATE" | "CREATE2" => { @@ -263,6 +264,7 @@ pub(crate) fn generate_jumpdest_table<'a>( } } } + info!("RETURN {}", &jumpdest_table); Ok(jumpdest_table) } From edab2056f980d18d9ac4f6612ef78ff70888128a Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 13 Nov 2024 01:44:57 +0100 Subject: [PATCH 104/112] remove offset --- evm_arithmetization/src/generation/prover_input.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 83fb5b18d..1d80de902 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -811,11 +811,10 @@ impl GenerationState { .unwrap_or(0) }) .unwrap_or(0) - + 6; - log::info!("TXOFFSET: {}", tx_in_batch_idx); - log::info!("TXNUM: {}", self.next_txn_index); - log::info!("TXLEN: {}", self.inputs.txn_hashes.len()); - log::info!("TX: {}", self.inputs.txn_number_before); + + 0; + log::info!("TXIDX: {}", tx_in_batch_idx); + log::info!("BATCH LEN: {}", self.inputs.txn_hashes.len()); + log::info!("TXN_NUM_BEFORE: {}", self.inputs.txn_number_before); let rpcw = self.inputs.jumpdest_table[tx_in_batch_idx].clone(); let rpcp: Option = rpcw.as_ref().map(|jdt| { get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, prev_max_wctx) From 5ee17da82a4d743aa0deef8a0b1d11843ebd88de Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 13 Nov 2024 15:55:27 +0100 Subject: [PATCH 105/112] . --- evm_arithmetization/src/generation/prover_input.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 1d80de902..5cfa0b36d 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -812,6 +812,7 @@ impl GenerationState { }) .unwrap_or(0) + 0; + log::info!("Maximum CTX in previous tx: {}", prev_max_wctx); log::info!("TXIDX: {}", tx_in_batch_idx); log::info!("BATCH LEN: {}", self.inputs.txn_hashes.len()); log::info!("TXN_NUM_BEFORE: {}", self.inputs.txn_number_before); From 71ecdf12c647219e43bae258cb732b87c5626c23 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 13 Nov 2024 19:37:13 +0100 Subject: [PATCH 106/112] debug info --- .../src/generation/prover_input.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 5cfa0b36d..11623d909 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -380,8 +380,18 @@ impl GenerationState { if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_ctx_mut(&batch_context) && let Some(next_jumpdest_address) = ctx_jumpdest_table.pop() { + log::info!( + "run_next_jumpdest_table_address, ctx {:>5}, address {:>5}", + batch_context, + next_jumpdest_address + 1 + ); Ok((next_jumpdest_address + 1).into()) } else { + log::info!( + "run_next_jumpdest_table_address, ctx {:>5}, address {:>5}", + batch_context, + 0 + ); jumpdest_table.remove_ctx(&batch_context); Ok(U256::zero()) } @@ -399,6 +409,11 @@ impl GenerationState { if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_ctx_mut(&context) && let Some(next_jumpdest_proof) = ctx_jumpdest_table.pop() { + log::info!( + "run_next_jumpdest_table_proof, ctx {:>5}, proof {:>5}", + context, + next_jumpdest_proof + ); Ok(next_jumpdest_proof.into()) } else { Err(ProgramError::ProverInputError( @@ -415,8 +430,18 @@ impl GenerationState { let address = u256_to_usize(stack_peek(self, 0)?)?; let closest_opcode_addr = get_closest_opcode_address(&code, address); Ok(if closest_opcode_addr < 32 { + log::info!( + "run_next_non_jumpdest_proof address {:>5}, closest_opcode_addr {:>5}, returns 0", + address, + closest_opcode_addr, + ); U256::zero() } else { + log::info!( + "run_next_non_jumpdest_proof address, {:>5}, closest_opcode_addr {:>5}", + address, + closest_opcode_addr, + ); closest_opcode_addr.into() }) } From 19a624cc0863a5c6700443111d8aa26e504cba5f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 14 Nov 2024 00:42:55 +0100 Subject: [PATCH 107/112] fix --- .../src/cpu/kernel/interpreter.rs | 5 ++-- .../kernel/tests/core/jumpdest_analysis.rs | 26 +++++++++++-------- .../src/generation/jumpdest.rs | 10 ++++++- .../src/generation/prover_input.rs | 26 ++++++++----------- .../src/generation/segments.rs | 2 +- evm_arithmetization/src/generation/state.rs | 8 +++--- zero/src/rpc/jumpdest.rs | 2 +- 7 files changed, 44 insertions(+), 35 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 0ba28834c..8f678716b 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -80,7 +80,7 @@ pub(crate) fn simulate_cpu_and_get_user_jumps( final_label: &str, state: &GenerationState, ) -> Option<(JumpDestTableProcessed, JumpDestTableWitness)> { - match state.jumpdest_table { + match state.jumpdest_tables { _ => { let halt_pc = KERNEL.global_labels[final_label]; let initial_context = state.registers.context; @@ -125,7 +125,7 @@ pub(crate) struct ExtraSegmentData { pub(crate) ger_prover_inputs: Vec, pub(crate) trie_root_ptrs: TrieRootPtrs, // todo - pub(crate) jumpdest_table: Option, + pub(crate) jumpdest_table: Vec>, pub(crate) access_lists_ptrs: LinkedListsPtrs, pub(crate) state_ptrs: LinkedListsPtrs, pub(crate) next_txn_index: usize, @@ -226,6 +226,7 @@ impl Interpreter { debug_inputs(inputs); let mut result = Self::new(initial_offset, initial_stack, max_cpu_len_log); + result.generation_state.jumpdest_tables = vec![None; inputs.batch_jumpdest_tables.len()]; result.initialize_interpreter_state(inputs); result } diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index a00ca27b4..b1b6d181d 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -18,7 +18,9 @@ use crate::witness::operation::CONTEXT_SCALING_FACTOR; impl Interpreter { pub(crate) fn set_jumpdest_analysis_inputs(&mut self, jumps: HashMap>) { let (jdtp, _jdtw) = self.generation_state.get_jumpdest_analysis_inputs(jumps); - self.generation_state.jumpdest_table = Some(jdtp); + + let tx_in_batch_idx = self.generation_state.next_txn_index - 1; + self.generation_state.jumpdest_tables[tx_in_batch_idx] = Some(jdtp); } pub(crate) fn get_jumpdest_bit(&self, offset: usize) -> U256 { @@ -103,9 +105,10 @@ fn test_jumpdest_analysis() -> Result<()> { ), )])); + let tx_in_batch_idx = interpreter.generation_state.next_txn_index - 1; // TODO The `set_jumpdest_analysis_inputs` method is never used. assert_eq!( - interpreter.generation_state.jumpdest_table, + interpreter.generation_state.jumpdest_tables[tx_in_batch_idx], // Context 3 has jumpdest 1, 5, 7. All have proof 0 and hence // the list [proof_0, jumpdest_0, ... ] is [0, 1, 0, 5, 0, 7, 8, 40] Some(JumpDestTableProcessed::new(HashMap::from([( @@ -126,11 +129,10 @@ fn test_jumpdest_analysis() -> Result<()> { .push(U256::from(CONTEXT) << CONTEXT_SCALING_FACTOR) .expect("The stack should not overflow"); + let tx_in_batch_idx = interpreter.generation_state.next_txn_index - 1; // We need to manually pop the jumpdest_table and push its value on the top of // the stack - interpreter - .generation_state - .jumpdest_table + interpreter.generation_state.jumpdest_tables[tx_in_batch_idx] .as_mut() .unwrap() .try_get_ctx_mut(&CONTEXT) @@ -180,9 +182,10 @@ fn test_packed_verification() -> Result<()> { let mut interpreter: Interpreter = Interpreter::new(write_table_if_jumpdest, initial_stack.clone(), None); interpreter.set_code(CONTEXT, code.clone()); - interpreter.generation_state.jumpdest_table = Some(JumpDestTableProcessed::new(HashMap::from( - [(3, vec![1, 33])], - ))); + let tx_in_batch_idx = interpreter.generation_state.next_txn_index - 1; + interpreter.generation_state.jumpdest_tables[tx_in_batch_idx] = Some( + JumpDestTableProcessed::new(HashMap::from([(3, vec![1, 33])])), + ); interpreter.run()?; @@ -195,9 +198,10 @@ fn test_packed_verification() -> Result<()> { let mut interpreter: Interpreter = Interpreter::new(write_table_if_jumpdest, initial_stack.clone(), None); interpreter.set_code(CONTEXT, code.clone()); - interpreter.generation_state.jumpdest_table = Some(JumpDestTableProcessed::new( - HashMap::from([(3, vec![1, 33])]), - )); + let tx_in_batch_idx = interpreter.generation_state.next_txn_index - 1; + interpreter.generation_state.jumpdest_tables[tx_in_batch_idx] = Some( + JumpDestTableProcessed::new(HashMap::from([(3, vec![1, 33])])), + ); assert!(interpreter.run().is_err()); diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 29ce6ffb2..de7dc5dd0 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -66,6 +66,10 @@ impl Context { context.insert(offset); }; } + + pub fn max_ctx(&self) -> usize { + self.keys().max().copied().unwrap_or(0) + } } impl JumpDestTableProcessed { @@ -90,7 +94,7 @@ impl JumpDestTableProcessed { } pub fn try_get_ctx_mut(&mut self, batch_ctx: &usize) -> Option<&mut Vec> { - log::info!("query_ctx {}", batch_ctx,); + // log::info!("query_ctx {}", batch_ctx,); self.witness_contexts.get_mut(batch_ctx) } @@ -192,6 +196,10 @@ impl JumpDestTableWitness { jdts.into_iter() .fold((Default::default(), 0), |(acc, cnt), t| acc.extend(t, cnt)) } + + pub fn max_ctx(&self) -> usize { + self.values().map(|ctx| ctx.max_ctx()).max().unwrap_or(0) + } } // The following Display instances are added to make it easier to read diffs. diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 11623d909..3e7228715 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -366,12 +366,13 @@ impl GenerationState { batch_context, self.next_txn_index - 1 ); + let tx_in_batch_idx = self.next_txn_index - 1; - if self.jumpdest_table.is_none() { + if self.jumpdest_tables[tx_in_batch_idx].is_none() { self.generate_jumpdest_table()?; } - let Some(jumpdest_table) = &mut self.jumpdest_table else { + let Some(jumpdest_table) = &mut self.jumpdest_tables[tx_in_batch_idx] else { return Err(ProgramError::ProverInputError( ProverInputError::InvalidJumpdestSimulation, )); @@ -400,7 +401,8 @@ impl GenerationState { /// Returns the proof for the last jump address. fn run_next_jumpdest_table_proof(&mut self) -> Result { let context = u256_to_usize(stack_peek(self, 1)? >> CONTEXT_SCALING_FACTOR)?; - let Some(jumpdest_table) = &mut self.jumpdest_table else { + let tx_in_batch_idx = self.next_txn_index - 1; + let Some(jumpdest_table) = &mut self.jumpdest_tables[tx_in_batch_idx] else { return Err(ProgramError::ProverInputError( ProverInputError::InvalidJumpdestSimulation, )); @@ -438,7 +440,7 @@ impl GenerationState { U256::zero() } else { log::info!( - "run_next_non_jumpdest_proof address, {:>5}, closest_opcode_addr {:>5}", + "run_next_non_jumpdest_proof address, {:>5}, closest_opcode_addr {:>5}", address, closest_opcode_addr, ); @@ -829,14 +831,8 @@ impl GenerationState { .get(tx_in_batch_idx - 1) .map(|x| x.as_ref()) .flatten() - .map(|jdt| { - jdt.iter() - .map(|(_h, jdt)| jdt.keys().max().copied().unwrap_or(0)) - .max() - .unwrap_or(0) - }) - .unwrap_or(0) - + 0; + .map(|x| x.max_ctx()) + .unwrap_or(0); log::info!("Maximum CTX in previous tx: {}", prev_max_wctx); log::info!("TXIDX: {}", tx_in_batch_idx); log::info!("BATCH LEN: {}", self.inputs.txn_hashes.len()); @@ -853,7 +849,7 @@ impl GenerationState { // } // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call - self.jumpdest_table = None; + self.jumpdest_tables[tx_in_batch_idx] = None; let (simp, simw) = simulate_cpu_and_get_user_jumps("terminate_common", &*self) .ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; // self.jumpdest_table = Some(simp.clone()); @@ -863,10 +859,10 @@ impl GenerationState { if rpcp.is_some() { dbg!(rpcp.as_ref(), Some(&simp)); // assert!(simp.is_subset(rpcp.as_ref().unwrap())); - self.jumpdest_table = rpcp; + self.jumpdest_tables[tx_in_batch_idx] = rpcp; } // self.jumpdest_table = rpcp; - self.jumpdest_table = Some(simp); + self.jumpdest_tables[tx_in_batch_idx] = Some(simp); return Ok(()); } diff --git a/evm_arithmetization/src/generation/segments.rs b/evm_arithmetization/src/generation/segments.rs index f5e6a85a5..6bdcaad32 100644 --- a/evm_arithmetization/src/generation/segments.rs +++ b/evm_arithmetization/src/generation/segments.rs @@ -82,7 +82,7 @@ fn build_segment_data( ger_prover_inputs: interpreter.generation_state.ger_prover_inputs.clone(), trie_root_ptrs: interpreter.generation_state.trie_root_ptrs.clone(), // todo verify - jumpdest_table: interpreter.generation_state.jumpdest_table.clone(), + jumpdest_table: interpreter.generation_state.jumpdest_tables.clone(), next_txn_index: interpreter.generation_state.next_txn_index, access_lists_ptrs: interpreter.generation_state.access_lists_ptrs.clone(), state_ptrs: interpreter.generation_state.state_ptrs.clone(), diff --git a/evm_arithmetization/src/generation/state.rs b/evm_arithmetization/src/generation/state.rs index 77aeb2400..c06f06dcc 100644 --- a/evm_arithmetization/src/generation/state.rs +++ b/evm_arithmetization/src/generation/state.rs @@ -390,7 +390,7 @@ pub struct GenerationState { /// the code (not necessarily pointing to an opcode) such that for every /// j in [i, i+32] it holds that code[j] < 0x7f - j + i. // jumpdest_table: Option, - pub(crate) jumpdest_table: Option, + pub(crate) jumpdest_tables: Vec>, /// Provides quick access to pointers that reference the location /// of either and account or a slot in the respective access list. pub(crate) access_lists_ptrs: LinkedListsPtrs, @@ -459,7 +459,7 @@ impl GenerationState { txn_root_ptr: 0, receipt_root_ptr: 0, }, - jumpdest_table: None, + jumpdest_tables: vec![], access_lists_ptrs: LinkedListsPtrs::default(), state_ptrs: LinkedListsPtrs::default(), ger_prover_inputs, @@ -575,7 +575,7 @@ impl GenerationState { txn_root_ptr: 0, receipt_root_ptr: 0, }, - jumpdest_table: self.jumpdest_table.clone(), + jumpdest_tables: self.jumpdest_tables.clone(), access_lists_ptrs: self.access_lists_ptrs.clone(), state_ptrs: self.state_ptrs.clone(), } @@ -593,7 +593,7 @@ impl GenerationState { self.trie_root_ptrs .clone_from(&segment_data.extra_data.trie_root_ptrs); // todo verify - self.jumpdest_table + self.jumpdest_tables .clone_from(&segment_data.extra_data.jumpdest_table); self.state_ptrs .clone_from(&segment_data.extra_data.state_ptrs); diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 57578bfad..3d64fc73c 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -193,7 +193,7 @@ pub(crate) fn generate_jumpdest_table<'a>( // exception or not. But this is of no consequence to the // generated Jumpdest table, so we can ignore the case. - jumpdest_table.insert(*code_hash, next_ctx_available, None); + // jumpdest_table.insert(*code_hash, next_ctx_available, None); next_ctx_available += 1; } "CREATE" | "CREATE2" => { From fde63a0c1c57caf6c91d7482ad294940aad9eb6f Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 14 Nov 2024 12:17:05 +0100 Subject: [PATCH 108/112] . --- .../src/cpu/kernel/interpreter.rs | 1 - .../src/generation/jumpdest.rs | 72 +------------------ 2 files changed, 2 insertions(+), 71 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 8f678716b..47b07aefc 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -182,7 +182,6 @@ pub(crate) fn get_jumpdest_analysis_inputs_rpc( let code = if code_map.contains_key(code_addr) { &code_map[code_addr] } else { - panic!("code not found"); &vec![] }; prove_context_jumpdests(code, ctx_jumpdests) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index de7dc5dd0..43f1a482f 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -23,7 +23,6 @@ //! [`JUMPDEST`]: https://www.evm.codes/?fork=cancun#5b use std::cmp::max; -use std::ops::Not as _; use std::{ collections::{BTreeSet, HashMap}, fmt::Display, @@ -44,10 +43,7 @@ pub struct Context(pub HashMap>); #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct JumpDestTableProcessed { witness_contexts: HashMap>, - /// Translates batch index to a wittness index - index: HashMap, - largest_batch_ctx: usize, - pub largest_witness_ctx: usize, + largest_witness_ctx: usize, } /// Map `CodeHash -> (Context -> [JumpDests])` @@ -76,19 +72,12 @@ impl JumpDestTableProcessed { pub fn new(ctx_map: HashMap>) -> Self { Self { witness_contexts: ctx_map, - // mapping from batch indices to witness indices - index: Default::default(), - largest_batch_ctx: 0, - largest_witness_ctx: 0, } } pub fn new_with_start(ctx_map: HashMap>, start_ctx: usize) -> Self { Self { witness_contexts: ctx_map, - // mapping from batch indices to witness indices - index: Default::default(), - largest_batch_ctx: 0, largest_witness_ctx: start_ctx, } } @@ -98,67 +87,9 @@ impl JumpDestTableProcessed { self.witness_contexts.get_mut(batch_ctx) } - // pub fn try_get_ctx_mut(&mut self, batch_ctx: &usize) -> Option<&mut - // Vec> { log::info!( - // "START: batch_ctx {} :: max_b {} :: max-w {} {:#?}", - // batch_ctx, - // self.largest_batch_ctx, - // self.largest_witness_ctx, - // self.index - // ); - - // if *batch_ctx <= self.largest_batch_ctx { - // let witness_ctx = self.index[batch_ctx]; - // return self.witness_contexts.get_mut(&witness_ctx); - // } - // self.largest_batch_ctx = *batch_ctx; - - // let mut new_witness_ctx = self.largest_witness_ctx; - // for i in (self.largest_witness_ctx + 1).. { - // if self.witness_contexts.contains_key(&i) { - // new_witness_ctx = i; - // break; - // } - // } - - // self.largest_witness_ctx = new_witness_ctx; - // self.index.insert(*batch_ctx, new_witness_ctx); - // log::info!( - // "END:{} {}->{} {:#?}", - // batch_ctx, - // self.largest_batch_ctx, - // self.largest_witness_ctx, - // self.index - // ); - - // self.witness_contexts.get_mut(&new_witness_ctx) - // } - - // pub fn remove_ctx(&mut self, batch_ctx: &usize) { - // let witness_index = self.index[batch_ctx]; - // self.witness_contexts.remove(&witness_index); - // } - pub fn remove_ctx(&mut self, batch_ctx: &usize) { self.witness_contexts.remove(&batch_ctx); } - - // pub fn last_ctx(self) -> usize { - // self.witness_contexts - // .keys() - // .max() - // .copied() - // .unwrap_or_default() - // } - - // pub fn is_subset(&self, other: &Self) -> bool { - // for (k, v) in self.witness_contexts.iter() { - // if other.witness_contexts.contains_key(k).not() || v != - // &other.witness_contexts[k] { return false; - // } - // } - // true - // } } impl JumpDestTableWitness { @@ -197,6 +128,7 @@ impl JumpDestTableWitness { .fold((Default::default(), 0), |(acc, cnt), t| acc.extend(t, cnt)) } + /// Obtain the context within any `code_hash` with maximal numeric value. pub fn max_ctx(&self) -> usize { self.values().map(|ctx| ctx.max_ctx()).max().unwrap_or(0) } From 515724493fcfd965512047d99e65c9bb0788b706 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 14 Nov 2024 17:29:50 +0100 Subject: [PATCH 109/112] premerge --- .../src/cpu/kernel/interpreter.rs | 7 ++- .../kernel/tests/core/jumpdest_analysis.rs | 2 +- .../src/generation/jumpdest.rs | 47 +++++++++++++------ .../src/generation/prover_input.rs | 33 +++++++------ zero/src/rpc/jumpdest.rs | 21 ++++++++- 5 files changed, 75 insertions(+), 35 deletions(-) diff --git a/evm_arithmetization/src/cpu/kernel/interpreter.rs b/evm_arithmetization/src/cpu/kernel/interpreter.rs index 47b07aefc..7d832d3b1 100644 --- a/evm_arithmetization/src/cpu/kernel/interpreter.rs +++ b/evm_arithmetization/src/cpu/kernel/interpreter.rs @@ -174,7 +174,7 @@ pub(crate) fn set_registers_and_run( pub(crate) fn get_jumpdest_analysis_inputs_rpc( jumpdest_table_rpc: &JumpDestTableWitness, code_map: &HashMap>, - prev: usize, + prev_max_batch_ctx: usize, ) -> JumpDestTableProcessed { let ctx_proofs = jumpdest_table_rpc .iter() @@ -187,7 +187,7 @@ pub(crate) fn get_jumpdest_analysis_inputs_rpc( prove_context_jumpdests(code, ctx_jumpdests) }) .collect(); - JumpDestTableProcessed::new_with_start(ctx_proofs, prev) + JumpDestTableProcessed::new_with_ctx_offset(ctx_proofs, prev_max_batch_ctx) } /// Orchestrates the proving of all contexts in a specific bytecode. @@ -202,8 +202,7 @@ pub(crate) fn get_jumpdest_analysis_inputs_rpc( /// Returns a [`HashMap`] from `ctx` to [`Vec`] of proofs. Each proofs ia a /// pair. fn prove_context_jumpdests(code: &[u8], ctx: &Context) -> HashMap> { - ctx.0 - .iter() + ctx.iter() .map(|(&ctx, jumpdests)| { let proofs = jumpdests.last().map_or(Vec::default(), |&largest_address| { get_proofs_and_jumpdests(code, largest_address, jumpdests.clone()) diff --git a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs index b1b6d181d..b047a4de1 100644 --- a/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs +++ b/evm_arithmetization/src/cpu/kernel/tests/core/jumpdest_analysis.rs @@ -135,7 +135,7 @@ fn test_jumpdest_analysis() -> Result<()> { interpreter.generation_state.jumpdest_tables[tx_in_batch_idx] .as_mut() .unwrap() - .try_get_ctx_mut(&CONTEXT) + .try_get_batch_ctx_mut(&CONTEXT) .unwrap() .pop(); interpreter diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 43f1a482f..48d6b613c 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -43,7 +43,7 @@ pub struct Context(pub HashMap>); #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize, Default)] pub(crate) struct JumpDestTableProcessed { witness_contexts: HashMap>, - largest_witness_ctx: usize, + ctx_offset: usize, } /// Map `CodeHash -> (Context -> [JumpDests])` @@ -51,9 +51,9 @@ pub(crate) struct JumpDestTableProcessed { pub struct JumpDestTableWitness(HashMap); impl Context { - pub fn get(&self, ctx: usize) -> Option<&BTreeSet> { - self.0.get(&ctx) - } + // pub fn get(&self, ctx: usize) -> Option<&BTreeSet> { + // self.get(&ctx) + // } pub fn insert(&mut self, ctx: usize, offset_opt: Option) { let context = self.entry(ctx).or_default(); @@ -63,7 +63,7 @@ impl Context { }; } - pub fn max_ctx(&self) -> usize { + pub fn max_batch_ctx(&self) -> usize { self.keys().max().copied().unwrap_or(0) } } @@ -72,23 +72,39 @@ impl JumpDestTableProcessed { pub fn new(ctx_map: HashMap>) -> Self { Self { witness_contexts: ctx_map, + ctx_offset: 0, } } - pub fn new_with_start(ctx_map: HashMap>, start_ctx: usize) -> Self { + pub fn new_with_ctx_offset(ctx_map: HashMap>, ctx_offset: usize) -> Self { Self { witness_contexts: ctx_map, - largest_witness_ctx: start_ctx, + ctx_offset, + } + } + + pub fn normalize(&self) -> Self { + let witness_contexts = self + .witness_contexts + .iter() + .map(|(ctx, offsets)| (ctx + self.ctx_offset, offsets.clone())) + .collect(); + + Self { + witness_contexts, + ctx_offset: 0, } } - pub fn try_get_ctx_mut(&mut self, batch_ctx: &usize) -> Option<&mut Vec> { - // log::info!("query_ctx {}", batch_ctx,); - self.witness_contexts.get_mut(batch_ctx) + pub fn try_get_batch_ctx_mut(&mut self, batch_ctx: &usize) -> Option<&mut Vec> { + log::info!("query_ctx {}", batch_ctx,); + let witness_context = *batch_ctx - self.ctx_offset; + self.witness_contexts.get_mut(&witness_context) } - pub fn remove_ctx(&mut self, batch_ctx: &usize) { - self.witness_contexts.remove(&batch_ctx); + pub fn remove_batch_ctx(&mut self, batch_ctx: &usize) { + let witness_context = *batch_ctx - self.ctx_offset; + self.witness_contexts.remove(&witness_context); } } @@ -129,8 +145,11 @@ impl JumpDestTableWitness { } /// Obtain the context within any `code_hash` with maximal numeric value. - pub fn max_ctx(&self) -> usize { - self.values().map(|ctx| ctx.max_ctx()).max().unwrap_or(0) + pub fn max_batch_ctx(&self) -> usize { + self.values() + .map(|ctx| ctx.max_batch_ctx()) + .max() + .unwrap_or(0) } } diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 3e7228715..d95bd8010 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -378,7 +378,7 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_ctx_mut(&batch_context) + if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_batch_ctx_mut(&batch_context) && let Some(next_jumpdest_address) = ctx_jumpdest_table.pop() { log::info!( @@ -393,7 +393,7 @@ impl GenerationState { batch_context, 0 ); - jumpdest_table.remove_ctx(&batch_context); + jumpdest_table.remove_batch_ctx(&batch_context); Ok(U256::zero()) } } @@ -408,7 +408,7 @@ impl GenerationState { )); }; - if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_ctx_mut(&context) + if let Some(ctx_jumpdest_table) = jumpdest_table.try_get_batch_ctx_mut(&context) && let Some(next_jumpdest_proof) = ctx_jumpdest_table.pop() { log::info!( @@ -825,26 +825,33 @@ impl GenerationState { /// respective contexts. fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { let tx_in_batch_idx = self.next_txn_index - 1; - let prev_max_wctx: usize = self + let prev_max_witness_ctx: usize = self .inputs .jumpdest_table .get(tx_in_batch_idx - 1) .map(|x| x.as_ref()) .flatten() - .map(|x| x.max_ctx()) + .map(|x| x.max_batch_ctx()) .unwrap_or(0); - log::info!("Maximum CTX in previous tx: {}", prev_max_wctx); + log::info!("Maximum CTX in previous tx: {}", prev_max_witness_ctx); log::info!("TXIDX: {}", tx_in_batch_idx); log::info!("BATCH LEN: {}", self.inputs.txn_hashes.len()); log::info!("TXN_NUM_BEFORE: {}", self.inputs.txn_number_before); + log::info!("TX HASH: {:#?}", self.inputs.txn_hashes); let rpcw = self.inputs.jumpdest_table[tx_in_batch_idx].clone(); - let rpcp: Option = rpcw.as_ref().map(|jdt| { - get_jumpdest_analysis_inputs_rpc(jdt, &self.inputs.contract_code, prev_max_wctx) - }); + let rpcp: Option = + rpcw.as_ref().map(|jdt: &JumpDestTableWitness| { + get_jumpdest_analysis_inputs_rpc( + jdt, + &self.inputs.contract_code, + prev_max_witness_ctx, + ) + .normalize() + }); log::info!("RPCW {:#?}", &rpcw); log::info!("RPCP {:#?}", &rpcp); // if rpcp.is_some() { - // self.jumpdest_table = rpcp; + // self.jumpdest_tables[tx_in_batch_idx] = rpcp; // return Ok(()); // } // Simulate the user's code and (unnecessarily) part of the kernel code, @@ -857,11 +864,9 @@ impl GenerationState { log::info!("SIMP {:#?}", &simp); if rpcp.is_some() { - dbg!(rpcp.as_ref(), Some(&simp)); - // assert!(simp.is_subset(rpcp.as_ref().unwrap())); - self.jumpdest_tables[tx_in_batch_idx] = rpcp; + dbg!(rpcp.as_ref().map(|x| x.normalize()), Some(&simp)); + assert_eq!(rpcp.as_ref(), Some(&simp)); } - // self.jumpdest_table = rpcp; self.jumpdest_tables[tx_in_batch_idx] = Some(simp); return Ok(()); } diff --git a/zero/src/rpc/jumpdest.rs b/zero/src/rpc/jumpdest.rs index 3d64fc73c..a03d8c8ca 100644 --- a/zero/src/rpc/jumpdest.rs +++ b/zero/src/rpc/jumpdest.rs @@ -133,7 +133,7 @@ pub(crate) fn generate_jumpdest_table<'a>( next_ctx_available += 1; let mut stuctlog_iter = structlog.iter().enumerate().peekable(); - while let Some((_step, entry)) = stuctlog_iter.next() { + while let Some((step, entry)) = stuctlog_iter.next() { let op = entry.op.as_str(); let curr_depth: usize = entry.depth.try_into().unwrap(); @@ -151,6 +151,22 @@ pub(crate) fn generate_jumpdest_table<'a>( info!("INSERT {} {}", *code_hash, *ctx); jumpdest_table.insert(*code_hash, *ctx, None); + // REVIEW: will be removed before merge + tracing::info!( + step, + curr_depth, + tx_hash = ?tx.hash, + ?code_hash, + ctx, + next_ctx_available, + pc = entry.pc, + pc_hex = format!("{:08x?}", entry.pc), + gas = entry.gas, + gas_cost = entry.gas_cost, + op, + ?entry, + ); + match op { "CALL" | "CALLCODE" | "DELEGATECALL" | "STATICCALL" => { prev_jump = None; @@ -193,7 +209,7 @@ pub(crate) fn generate_jumpdest_table<'a>( // exception or not. But this is of no consequence to the // generated Jumpdest table, so we can ignore the case. - // jumpdest_table.insert(*code_hash, next_ctx_available, None); + jumpdest_table.insert(*code_hash, next_ctx_available, None); next_ctx_available += 1; } "CREATE" | "CREATE2" => { @@ -257,6 +273,7 @@ pub(crate) fn generate_jumpdest_table<'a>( } "EXTCODECOPY" | "EXTCODESIZE" => { prev_jump = None; + jumpdest_table.insert(*code_hash, next_ctx_available, None); next_ctx_available += 1; } _ => { From cb9847882d7612c05fe5738380f5ca326515f429 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Thu, 14 Nov 2024 20:18:11 +0100 Subject: [PATCH 110/112] 748 batched passing --- .../src/generation/prover_input.rs | 19 ++++++++++--------- trace_decoder/src/core.rs | 6 ------ zero/src/rpc/native/txn.rs | 1 - 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index d95bd8010..07baacdf2 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -837,7 +837,8 @@ impl GenerationState { log::info!("TXIDX: {}", tx_in_batch_idx); log::info!("BATCH LEN: {}", self.inputs.txn_hashes.len()); log::info!("TXN_NUM_BEFORE: {}", self.inputs.txn_number_before); - log::info!("TX HASH: {:#?}", self.inputs.txn_hashes); + log::info!("TX BATCH: {:#?}", self.inputs.txn_hashes); + log::info!("TX HASH: {:#?}", self.inputs.txn_hashes[tx_in_batch_idx]); let rpcw = self.inputs.jumpdest_table[tx_in_batch_idx].clone(); let rpcp: Option = rpcw.as_ref().map(|jdt: &JumpDestTableWitness| { @@ -850,10 +851,10 @@ impl GenerationState { }); log::info!("RPCW {:#?}", &rpcw); log::info!("RPCP {:#?}", &rpcp); - // if rpcp.is_some() { - // self.jumpdest_tables[tx_in_batch_idx] = rpcp; - // return Ok(()); - // } + if rpcp.is_some() { + self.jumpdest_tables[tx_in_batch_idx] = rpcp; + return Ok(()); + } // Simulate the user's code and (unnecessarily) part of the kernel code, // skipping the validate table call self.jumpdest_tables[tx_in_batch_idx] = None; @@ -863,10 +864,10 @@ impl GenerationState { log::info!("SIMW {:#?}", &simw); log::info!("SIMP {:#?}", &simp); - if rpcp.is_some() { - dbg!(rpcp.as_ref().map(|x| x.normalize()), Some(&simp)); - assert_eq!(rpcp.as_ref(), Some(&simp)); - } + // if let Some(rpcp) = rpcp { + // dbg!(rpcp.as_ref().map(|x| x.normalize()), Some(&simp)); + // assert_eq!(rpcp.is_super(&simp)); + // } self.jumpdest_tables[tx_in_batch_idx] = Some(simp); return Ok(()); } diff --git a/trace_decoder/src/core.rs b/trace_decoder/src/core.rs index 94758a651..e0abcd30f 100644 --- a/trace_decoder/src/core.rs +++ b/trace_decoder/src/core.rs @@ -221,12 +221,6 @@ pub fn entrypoint( // the whole batch. There is an optimization opportunity // here. dbg!(&jumpdest_tables); - // let res = jumpdest_tables - // .into_iter() - // .collect::>>() - // .map(|jdt| JumpDestTableWitness::merge(jdt.iter()).0); - // dbg!(&res); - if jumpdest_tables.iter().any(Option::is_none) { repeat(None).take(jumpdest_tables.len()).collect::>() } else { diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 62086c0d1..daf3d6388 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -30,7 +30,6 @@ use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use tracing::{debug, warn}; use crate::rpc::jumpdest::get_block_normalized_structlogs; -// use crate::rpc::Compat; use crate::rpc::{ jumpdest::{self}, JumpdestSrc, From e47731d6096b24db99b500fe4f7c5e15f06c3569 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 15 Nov 2024 15:42:05 +0100 Subject: [PATCH 111/112] n=18 ok --- .../src/generation/jumpdest.rs | 4 ++++ .../src/generation/prover_input.rs | 21 ++++++++++--------- zero/src/bin/rpc.rs | 1 + 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/evm_arithmetization/src/generation/jumpdest.rs b/evm_arithmetization/src/generation/jumpdest.rs index 48d6b613c..dc180c74d 100644 --- a/evm_arithmetization/src/generation/jumpdest.rs +++ b/evm_arithmetization/src/generation/jumpdest.rs @@ -106,6 +106,10 @@ impl JumpDestTableProcessed { let witness_context = *batch_ctx - self.ctx_offset; self.witness_contexts.remove(&witness_context); } + + pub fn max_batch_ctx(&self) -> usize { + self.witness_contexts.keys().max().copied().unwrap_or(0) + } } impl JumpDestTableWitness { diff --git a/evm_arithmetization/src/generation/prover_input.rs b/evm_arithmetization/src/generation/prover_input.rs index 07baacdf2..efeefc563 100644 --- a/evm_arithmetization/src/generation/prover_input.rs +++ b/evm_arithmetization/src/generation/prover_input.rs @@ -826,18 +826,17 @@ impl GenerationState { fn generate_jumpdest_table(&mut self) -> Result<(), ProgramError> { let tx_in_batch_idx = self.next_txn_index - 1; let prev_max_witness_ctx: usize = self - .inputs - .jumpdest_table + .jumpdest_tables .get(tx_in_batch_idx - 1) .map(|x| x.as_ref()) .flatten() .map(|x| x.max_batch_ctx()) .unwrap_or(0); - log::info!("Maximum CTX in previous tx: {}", prev_max_witness_ctx); - log::info!("TXIDX: {}", tx_in_batch_idx); + log::info!("TX BATCH: {:#?}", self.inputs.txn_hashes); log::info!("BATCH LEN: {}", self.inputs.txn_hashes.len()); log::info!("TXN_NUM_BEFORE: {}", self.inputs.txn_number_before); - log::info!("TX BATCH: {:#?}", self.inputs.txn_hashes); + log::info!("Maximum CTX in previous tx: {}", prev_max_witness_ctx); + log::info!("TXIDX: {}", tx_in_batch_idx); log::info!("TX HASH: {:#?}", self.inputs.txn_hashes[tx_in_batch_idx]); let rpcw = self.inputs.jumpdest_table[tx_in_batch_idx].clone(); let rpcp: Option = @@ -860,14 +859,16 @@ impl GenerationState { self.jumpdest_tables[tx_in_batch_idx] = None; let (simp, simw) = simulate_cpu_and_get_user_jumps("terminate_common", &*self) .ok_or(ProgramError::ProverInputError(InvalidJumpdestSimulation))?; - // self.jumpdest_table = Some(simp.clone()); log::info!("SIMW {:#?}", &simw); log::info!("SIMP {:#?}", &simp); - // if let Some(rpcp) = rpcp { - // dbg!(rpcp.as_ref().map(|x| x.normalize()), Some(&simp)); - // assert_eq!(rpcp.is_super(&simp)); - // } + if let Some(rpcp) = rpcp { + if &rpcp != &simp { + log::warn!("MISMATCH"); + dbg!(Some(&rpcp), Some(&simp)); + } + } + self.jumpdest_tables[tx_in_batch_idx] = Some(simp); return Ok(()); } diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index 5ca8e46a4..5186c668d 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -237,6 +237,7 @@ async fn main() -> anyhow::Result<()> { // With the default configuration trace information is written // to stdout, but we already use stdout to write our payload (the witness). .with_writer(std::io::stderr) + // .json() .with_ansi(false) .with_filter(EnvFilter::from_default_env()), ) From 55a5dd815400093819798824456e6f97bb351496 Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Fri, 15 Nov 2024 16:34:51 +0100 Subject: [PATCH 112/112] test batching --- test_batching.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 test_batching.sh diff --git a/test_batching.sh b/test_batching.sh new file mode 100755 index 000000000..26a06985d --- /dev/null +++ b/test_batching.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -exo + +test_batch_size() { + cargo --quiet run --release --bin leader -- --runtime in-memory -b $2 -n 1 --test-only --save-inputs-on-error stdio < $1.witness.json +} + +main() { + local NUM_TX=$(expr $(cast block $1 | wc -l) - 10) + for BATCH_SIZE in $(seq $NUM_TX) + do + test_batch_size $1 $BATCH_SIZE + done +} + +main 748