From 7e6d11d27b7720de53ed57e2d66afcd38ace9f14 Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 27 Feb 2025 19:16:31 -0500 Subject: [PATCH 1/4] adds `ItemSize` trait in `tower-batch-control` --- tower-batch-control/src/layer.rs | 4 +++- tower-batch-control/src/lib.rs | 17 +++++++++++++++-- tower-batch-control/src/service.rs | 12 +++++++----- tower-batch-control/src/worker.rs | 8 +++++--- tower-batch-control/tests/ed25519.rs | 25 ++++++++++++++++++++----- 5 files changed, 50 insertions(+), 16 deletions(-) diff --git a/tower-batch-control/src/layer.rs b/tower-batch-control/src/layer.rs index c3757eef663..f98ebdcdafd 100644 --- a/tower-batch-control/src/layer.rs +++ b/tower-batch-control/src/layer.rs @@ -5,6 +5,8 @@ use std::{fmt, marker::PhantomData}; use tower::layer::Layer; use tower::Service; +use crate::ItemSize; + use super::{service::Batch, BatchControl}; /// Adds a layer performing batch processing of requests. @@ -43,7 +45,7 @@ impl BatchLayer { } } -impl Layer for BatchLayer +impl Layer for BatchLayer where S: Service> + Send + 'static, S::Future: Send, diff --git a/tower-batch-control/src/lib.rs b/tower-batch-control/src/lib.rs index 2628a09562e..d8f977fa279 100644 --- a/tower-batch-control/src/lib.rs +++ b/tower-batch-control/src/lib.rs @@ -105,14 +105,14 @@ type BoxError = Box; /// Signaling mechanism for batchable services that allows explicit flushing. /// /// This request type is a generic wrapper for the inner `Req` type. -pub enum BatchControl { +pub enum BatchControl { /// A new batch item. Item(Req), /// The current batch should be flushed. Flush, } -impl From for BatchControl { +impl From for BatchControl { fn from(req: Req) -> BatchControl { BatchControl::Item(req) } @@ -120,3 +120,16 @@ impl From for BatchControl { pub use self::layer::BatchLayer; pub use self::service::Batch; + +/// A trait for reading the size of a request to [`BatchControl`] services. +pub trait ItemSize { + /// Returns the size of this item relative to the maximum threshold for flushing + /// requests to the underlying service. + fn item_size(&self) -> usize { + 1 + } +} + +// [`ItemSize`] impls for test `Item` types +impl ItemSize for () {} +impl ItemSize for &'static str {} diff --git a/tower-batch-control/src/service.rs b/tower-batch-control/src/service.rs index 414e2452529..8ae3f4dc64a 100644 --- a/tower-batch-control/src/service.rs +++ b/tower-batch-control/src/service.rs @@ -19,6 +19,8 @@ use tokio_util::sync::PollSemaphore; use tower::Service; use tracing::{info_span, Instrument}; +use crate::ItemSize; + use super::{ future::ResponseFuture, message::Message, @@ -34,7 +36,7 @@ pub const QUEUE_BATCH_LIMIT: usize = 64; /// Allows batch processing of requests. /// /// See the crate documentation for more details. -pub struct Batch +pub struct Batch where T: Service>, { @@ -72,7 +74,7 @@ where worker_handle: Arc>>>, } -impl fmt::Debug for Batch +impl fmt::Debug for Batch where T: Service>, { @@ -88,7 +90,7 @@ where } } -impl Batch +impl Batch where T: Service>, T::Future: Send + 'static, @@ -218,7 +220,7 @@ where } } -impl Service for Batch +impl Service for Batch where T: Service>, T::Future: Send + 'static, @@ -322,7 +324,7 @@ where } } -impl Clone for Batch +impl Clone for Batch where T: Service>, { diff --git a/tower-batch-control/src/worker.rs b/tower-batch-control/src/worker.rs index 865a2f37009..7bf8fd55786 100644 --- a/tower-batch-control/src/worker.rs +++ b/tower-batch-control/src/worker.rs @@ -19,6 +19,8 @@ use tokio_util::sync::PollSemaphore; use tower::{Service, ServiceExt}; use tracing_futures::Instrument; +use crate::ItemSize; + use super::{ error::{Closed, ServiceError}, message::{self, Message}, @@ -34,7 +36,7 @@ use super::{ /// implement (only call). #[pin_project(PinnedDrop)] #[derive(Debug)] -pub struct Worker +pub struct Worker where T: Service>, T::Future: Send + 'static, @@ -91,7 +93,7 @@ pub(crate) struct ErrorHandle { inner: Arc>>, } -impl Worker +impl Worker where T: Service>, T::Future: Send + 'static, @@ -351,7 +353,7 @@ impl Clone for ErrorHandle { } #[pin_project::pinned_drop] -impl PinnedDrop for Worker +impl PinnedDrop for Worker where T: Service>, T::Future: Send + 'static, diff --git a/tower-batch-control/tests/ed25519.rs b/tower-batch-control/tests/ed25519.rs index 5bbee2f72fb..70bb191474b 100644 --- a/tower-batch-control/tests/ed25519.rs +++ b/tower-batch-control/tests/ed25519.rs @@ -8,14 +8,14 @@ use std::{ }; use color_eyre::{eyre::eyre, Report}; -use ed25519_zebra::{batch, Error, SigningKey, VerificationKeyBytes}; +use ed25519_zebra::{batch, Error, Signature, SigningKey, VerificationKeyBytes}; use futures::stream::{FuturesOrdered, StreamExt}; use futures::FutureExt; use futures_core::Future; use rand::thread_rng; use tokio::sync::{oneshot::error::RecvError, watch}; use tower::{Service, ServiceExt}; -use tower_batch_control::{Batch, BatchControl}; +use tower_batch_control::{Batch, BatchControl, ItemSize}; use tower_fallback::Fallback; // ============ service impl ============ @@ -33,8 +33,23 @@ type VerifyResult = Result<(), Error>; type Sender = watch::Sender>; /// The type of the batch item. -/// This is an `Ed25519Item`. -type Item = batch::Item; +/// This is a newtype around a `Ed25519Item`. +#[derive(Clone, Debug)] +struct Item(batch::Item); + +impl ItemSize for Item {} + +impl<'msg, M: AsRef<[u8]> + ?Sized> From<(VerificationKeyBytes, Signature, &'msg M)> for Item { + fn from(tup: (VerificationKeyBytes, Signature, &'msg M)) -> Self { + Self(batch::Item::from(tup)) + } +} + +impl Item { + fn verify_single(self) -> VerifyResult { + self.0.verify_single() + } +} /// Ed25519 signature verifier service struct Verifier { @@ -106,7 +121,7 @@ impl Service> for Verifier { fn call(&mut self, req: BatchControl) -> Self::Future { match req { - BatchControl::Item(item) => { + BatchControl::Item(Item(item)) => { tracing::trace!("got ed25519 item"); self.batch.queue(item); let mut rx = self.tx.subscribe(); From edde61e42c10b572183c8f2e3f80f1767f82210d Mon Sep 17 00:00:00 2001 From: Arya Date: Thu, 27 Feb 2025 19:59:11 -0500 Subject: [PATCH 2/4] Replaces batch proof verifier `Item` type aliases with newtypes and impls `ItemSIze` for them. --- tower-batch-control/tests/ed25519.rs | 2 +- zebra-consensus/src/primitives/ed25519.rs | 27 ++++++++++-- zebra-consensus/src/primitives/groth16.rs | 24 +++++++++-- .../src/primitives/groth16/tests.rs | 41 ++++++++----------- zebra-consensus/src/primitives/halo2.rs | 8 +++- zebra-consensus/src/primitives/redjubjub.rs | 23 +++++++++-- zebra-consensus/src/primitives/redpallas.rs | 41 +++++++++++++++++-- 7 files changed, 126 insertions(+), 40 deletions(-) diff --git a/tower-batch-control/tests/ed25519.rs b/tower-batch-control/tests/ed25519.rs index 70bb191474b..aeb911f7e42 100644 --- a/tower-batch-control/tests/ed25519.rs +++ b/tower-batch-control/tests/ed25519.rs @@ -33,7 +33,7 @@ type VerifyResult = Result<(), Error>; type Sender = watch::Sender>; /// The type of the batch item. -/// This is a newtype around a `Ed25519Item`. +/// This is a newtype around an `Ed25519Item`. #[derive(Clone, Debug)] struct Item(batch::Item); diff --git a/zebra-consensus/src/primitives/ed25519.rs b/zebra-consensus/src/primitives/ed25519.rs index 10926db67eb..b97b983b735 100644 --- a/zebra-consensus/src/primitives/ed25519.rs +++ b/zebra-consensus/src/primitives/ed25519.rs @@ -13,7 +13,7 @@ use rand::thread_rng; use tokio::sync::watch; use tower::{util::ServiceFn, Service}; -use tower_batch_control::{Batch, BatchControl}; +use tower_batch_control::{Batch, BatchControl, ItemSize}; use tower_fallback::Fallback; use zebra_chain::primitives::ed25519::*; @@ -34,8 +34,29 @@ type VerifyResult = Result<(), Error>; type Sender = watch::Sender>; /// The type of the batch item. -/// This is an `Ed25519Item`. -pub type Item = batch::Item; +/// This is a newtype around an `Ed25519Item`. +#[derive(Clone, Debug)] +pub struct Item(batch::Item); + +impl ItemSize for Item {} + +impl<'msg, M: AsRef<[u8]> + ?Sized> From<(VerificationKeyBytes, Signature, &'msg M)> for Item { + fn from(tup: (VerificationKeyBytes, Signature, &'msg M)) -> Self { + Self(batch::Item::from(tup)) + } +} + +impl From for batch::Item { + fn from(Item(item): Item) -> Self { + item + } +} + +impl Item { + fn verify_single(self) -> VerifyResult { + self.0.verify_single() + } +} /// Global batch verification context for Ed25519 signatures. /// diff --git a/zebra-consensus/src/primitives/groth16.rs b/zebra-consensus/src/primitives/groth16.rs index e2ec64144fb..ee4f54e290e 100644 --- a/zebra-consensus/src/primitives/groth16.rs +++ b/zebra-consensus/src/primitives/groth16.rs @@ -21,7 +21,7 @@ use rand::thread_rng; use tokio::sync::watch; use tower::{util::ServiceFn, Service}; -use tower_batch_control::{Batch, BatchControl}; +use tower_batch_control::{Batch, BatchControl, ItemSize}; use tower_fallback::{BoxedError, Fallback}; use zebra_chain::{ @@ -57,8 +57,24 @@ type VerifyResult = Result<(), VerificationError>; type Sender = watch::Sender>; /// The type of the batch item. -/// This is a Groth16 verification item. -pub type Item = batch::Item; +/// This is a newtype around a Groth16 verification item. +#[derive(Clone, Debug)] +pub struct Item(batch::Item); + +impl ItemSize for Item {} + +impl>> From for Item { + fn from(value: T) -> Self { + Self(value.into()) + } +} + +impl Item { + /// Convenience method to call a method on the inner value to perform non-batched verification. + pub fn verify_single(self, pvk: &PreparedVerifyingKey) -> VerifyResult { + self.0.verify_single(pvk) + } +} /// The type of a raw verifying key. /// This is the key used to verify batches. @@ -469,7 +485,7 @@ impl Service> for Verifier { match req { BatchControl::Item(item) => { tracing::trace!("got item"); - self.batch.queue(item); + self.batch.queue(item.0); let mut rx = self.tx.subscribe(); Box::pin(async move { diff --git a/zebra-consensus/src/primitives/groth16/tests.rs b/zebra-consensus/src/primitives/groth16/tests.rs index 52c00d575a3..f40ccc8ac1c 100644 --- a/zebra-consensus/src/primitives/groth16/tests.rs +++ b/zebra-consensus/src/primitives/groth16/tests.rs @@ -21,11 +21,10 @@ async fn verify_groth16_spends_and_outputs( ) -> Result<(), V::Error> where V: tower::Service, - >>::Error: - std::fmt::Debug - + std::convert::From< - std::boxed::Box, - >, + >::Error: std::fmt::Debug + + std::convert::From< + std::boxed::Box, + >, { let _init_guard = zebra_test::init(); @@ -133,10 +132,9 @@ async fn verify_invalid_groth16_output_description( ) -> Result<(), V::Error> where V: tower::Service, - >>::Error: - std::convert::From< - std::boxed::Box, - >, + >::Error: std::convert::From< + std::boxed::Box, + >, { let _init_guard = zebra_test::init(); @@ -213,11 +211,10 @@ async fn verify_groth16_joinsplits( ) -> Result<(), V::Error> where V: tower::Service, - >>::Error: - std::fmt::Debug - + std::convert::From< - std::boxed::Box, - >, + >::Error: std::fmt::Debug + + std::convert::From< + std::boxed::Box, + >, { let _init_guard = zebra_test::init(); @@ -285,11 +282,10 @@ async fn verify_groth16_joinsplit_vector( ) -> Result<(), V::Error> where V: tower::Service, - >>::Error: - std::fmt::Debug - + std::convert::From< - std::boxed::Box, - >, + >::Error: std::fmt::Debug + + std::convert::From< + std::boxed::Box, + >, { let _init_guard = zebra_test::init(); @@ -402,10 +398,9 @@ async fn verify_invalid_groth16_joinsplit_description( ) -> Result<(), V::Error> where V: tower::Service, - >>::Error: - std::convert::From< - std::boxed::Box, - >, + >::Error: std::convert::From< + std::boxed::Box, + >, { let _init_guard = zebra_test::init(); diff --git a/zebra-consensus/src/primitives/halo2.rs b/zebra-consensus/src/primitives/halo2.rs index ffc58a5feb8..79ad10699d5 100644 --- a/zebra-consensus/src/primitives/halo2.rs +++ b/zebra-consensus/src/primitives/halo2.rs @@ -16,7 +16,7 @@ use rand::{thread_rng, CryptoRng, RngCore}; use thiserror::Error; use tokio::sync::watch; use tower::{util::ServiceFn, Service}; -use tower_batch_control::{Batch, BatchControl}; +use tower_batch_control::{Batch, BatchControl, ItemSize}; use tower_fallback::Fallback; use crate::BoxError; @@ -92,6 +92,12 @@ pub struct Item { proof: orchard::circuit::Proof, } +impl ItemSize for Item { + fn item_size(&self) -> usize { + self.instances.len() + } +} + impl Item { /// Perform non-batched verification of this `Item`. /// diff --git a/zebra-consensus/src/primitives/redjubjub.rs b/zebra-consensus/src/primitives/redjubjub.rs index 3958647b57c..8e07b81ca90 100644 --- a/zebra-consensus/src/primitives/redjubjub.rs +++ b/zebra-consensus/src/primitives/redjubjub.rs @@ -13,7 +13,7 @@ use rand::thread_rng; use tokio::sync::watch; use tower::{util::ServiceFn, Service}; -use tower_batch_control::{Batch, BatchControl}; +use tower_batch_control::{Batch, BatchControl, ItemSize}; use tower_fallback::Fallback; use zebra_chain::primitives::redjubjub::*; @@ -35,8 +35,23 @@ type VerifyResult = Result<(), Error>; type Sender = watch::Sender>; /// The type of the batch item. -/// This is a `RedJubjubItem`. -pub type Item = batch::Item; +/// This is a newtype around a `RedJubjubItem`. +#[derive(Clone, Debug)] +pub struct Item(batch::Item); + +impl ItemSize for Item {} + +impl> From for Item { + fn from(value: T) -> Self { + Self(value.into()) + } +} + +impl Item { + fn verify_single(self) -> VerifyResult { + self.0.verify_single() + } +} /// Global batch verification context for RedJubjub signatures. /// @@ -152,7 +167,7 @@ impl Service> for Verifier { match req { BatchControl::Item(item) => { tracing::trace!("got item"); - self.batch.queue(item); + self.batch.queue(item.0); let mut rx = self.tx.subscribe(); Box::pin(async move { diff --git a/zebra-consensus/src/primitives/redpallas.rs b/zebra-consensus/src/primitives/redpallas.rs index d848682aaa4..4daff763091 100644 --- a/zebra-consensus/src/primitives/redpallas.rs +++ b/zebra-consensus/src/primitives/redpallas.rs @@ -13,10 +13,10 @@ use rand::thread_rng; use tokio::sync::watch; use tower::{util::ServiceFn, Service}; -use tower_batch_control::{Batch, BatchControl}; +use tower_batch_control::{Batch, BatchControl, ItemSize}; use tower_fallback::Fallback; -use zebra_chain::primitives::reddsa::{batch, orchard, Error}; +use zebra_chain::primitives::reddsa::{batch, orchard, Error, Signature, VerificationKeyBytes}; use crate::BoxError; @@ -35,8 +35,41 @@ type VerifyResult = Result<(), Error>; type Sender = watch::Sender>; /// The type of the batch item. -/// This is a `RedPallasItem`. -pub type Item = batch::Item; +/// This is a newtype around a `RedPallasItem`. +#[derive(Clone, Debug)] +pub struct Item(batch::Item); + +impl ItemSize for Item {} + +impl From for batch::Item { + fn from(Item(item): Item) -> Self { + item + } +} + +impl Item { + fn verify_single(self) -> VerifyResult { + self.0.verify_single() + } + + /// Create a batch item from a `SpendAuth` signature. + pub fn from_spendauth( + vk_bytes: VerificationKeyBytes, + sig: Signature, + msg: &impl AsRef<[u8]>, + ) -> Self { + Self(batch::Item::from_spendauth(vk_bytes, sig, msg)) + } + + /// Create a batch item from a `Binding` signature. + pub fn from_binding( + vk_bytes: VerificationKeyBytes, + sig: Signature, + msg: &impl AsRef<[u8]>, + ) -> Self { + Self(batch::Item::from_binding(vk_bytes, sig, msg)) + } +} /// Global batch verification context for RedPallas signatures. /// From 5bf021965160b529e968b9e43bdc878090a116d6 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 28 Feb 2025 00:50:18 -0500 Subject: [PATCH 3/4] (draft) Replaces orchard proof verifiers with orchard batch validator --- Cargo.lock | 1 + Cargo.toml | 2 + zebra-chain/src/orchard/note/ciphertexts.rs | 11 + zebra-chain/src/orchard/note/nullifiers.rs | 13 ++ zebra-consensus/Cargo.toml | 2 + zebra-consensus/src/primitives.rs | 7 +- zebra-consensus/src/primitives/halo2.rs | 193 ++++++++++-------- zebra-consensus/src/primitives/halo2/tests.rs | 18 +- zebra-consensus/src/transaction.rs | 77 +------ 9 files changed, 153 insertions(+), 171 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9095e4f1ee5..fba0733cb8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6049,6 +6049,7 @@ dependencies = [ "jubjub", "lazy_static", "metrics", + "nonempty", "num-integer", "once_cell", "orchard", diff --git a/Cargo.toml b/Cargo.toml index 6aef936bc39..3af161aa736 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,6 +90,7 @@ metrics-exporter-prometheus = { version = "0.16.1", default-features = false } mset = "0.1.1" nix = "0.29.0" num-integer = "0.1.46" +nonempty = "0.7.0" once_cell = "1.20.2" ordered-map = "0.4.2" owo-colors = "4.1.0" @@ -152,6 +153,7 @@ x25519-dalek = "2.0.1" zcash_note_encryption = "0.4.1" zcash_script = "0.2.0" + [workspace.metadata.release] # We always do releases from the main branch diff --git a/zebra-chain/src/orchard/note/ciphertexts.rs b/zebra-chain/src/orchard/note/ciphertexts.rs index 8f857cf1444..1870dfc6671 100644 --- a/zebra-chain/src/orchard/note/ciphertexts.rs +++ b/zebra-chain/src/orchard/note/ciphertexts.rs @@ -44,6 +44,11 @@ impl From for [u8; 580] { enc_ciphertext.0 } } +impl From<&EncryptedNote> for [u8; 580] { + fn from(&EncryptedNote(enc_ciphertext): &EncryptedNote) -> Self { + enc_ciphertext + } +} impl PartialEq for EncryptedNote { fn eq(&self, other: &Self) -> bool { @@ -102,6 +107,12 @@ impl From for [u8; 80] { } } +impl From<&WrappedNoteKey> for [u8; 80] { + fn from(out_ciphertext: &WrappedNoteKey) -> Self { + out_ciphertext.0 + } +} + impl PartialEq for WrappedNoteKey { fn eq(&self, other: &Self) -> bool { self.0[..] == other.0[..] diff --git a/zebra-chain/src/orchard/note/nullifiers.rs b/zebra-chain/src/orchard/note/nullifiers.rs index e9a9109a5e8..62fc6446570 100644 --- a/zebra-chain/src/orchard/note/nullifiers.rs +++ b/zebra-chain/src/orchard/note/nullifiers.rs @@ -10,6 +10,19 @@ use crate::serialization::{serde_helpers, SerializationError}; #[derive(Clone, Copy, Debug, Eq, Serialize, Deserialize)] pub struct Nullifier(#[serde(with = "serde_helpers::Base")] pub(crate) pallas::Base); +impl From for orchard::note::Nullifier { + fn from(value: Nullifier) -> Self { + // TODO: impl From for orchard::note::Nullifier in the orchard repo + Self::from_bytes(&value.into()).expect("should be valid") + } +} + +impl From<&Nullifier> for orchard::note::Nullifier { + fn from(&value: &Nullifier) -> Self { + Self::from(value) + } +} + impl Hash for Nullifier { fn hash(&self, state: &mut H) { self.0.to_repr().hash(state); diff --git a/zebra-consensus/Cargo.toml b/zebra-consensus/Cargo.toml index 7fe94c1b380..0d752c66e28 100644 --- a/zebra-consensus/Cargo.toml +++ b/zebra-consensus/Cargo.toml @@ -71,6 +71,8 @@ zebra-state = { path = "../zebra-state", version = "1.0.0-beta.45" } zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.45" } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.45" } +nonempty.workspace = true + # prod feature progress-bar howudoin = { workspace = true, optional = true } diff --git a/zebra-consensus/src/primitives.rs b/zebra-consensus/src/primitives.rs index e3ab3a4f865..f6e7099d95e 100644 --- a/zebra-consensus/src/primitives.rs +++ b/zebra-consensus/src/primitives.rs @@ -33,12 +33,9 @@ pub async fn spawn_fifo_and_convert< } /// Fires off a task into the Rayon threadpool and awaits the result through a oneshot channel. -pub async fn spawn_fifo< - E: 'static + std::error::Error + Sync + Send, - F: 'static + FnOnce() -> Result<(), E> + Send, ->( +pub async fn spawn_fifo T + Send>( f: F, -) -> Result, RecvError> { +) -> Result { // Rayon doesn't have a spawn function that returns a value, // so we use a oneshot channel instead. let (rsp_tx, rsp_rx) = tokio::sync::oneshot::channel(); diff --git a/zebra-consensus/src/primitives/halo2.rs b/zebra-consensus/src/primitives/halo2.rs index 79ad10699d5..7f59c16d554 100644 --- a/zebra-consensus/src/primitives/halo2.rs +++ b/zebra-consensus/src/primitives/halo2.rs @@ -9,22 +9,27 @@ use std::{ }; use futures::{future::BoxFuture, FutureExt}; +use nonempty::NonEmpty; use once_cell::sync::Lazy; -use orchard::circuit::VerifyingKey; -use rand::{thread_rng, CryptoRng, RngCore}; +use orchard::{ + bundle::BatchValidator, + circuit::VerifyingKey, + note::{ExtractedNoteCommitment, TransmittedNoteCiphertext}, +}; +use rand::thread_rng; +use zebra_chain::{amount::Amount, transaction::SigHash}; +use crate::BoxError; use thiserror::Error; use tokio::sync::watch; use tower::{util::ServiceFn, Service}; use tower_batch_control::{Batch, BatchControl, ItemSize}; use tower_fallback::Fallback; -use crate::BoxError; - -use super::{spawn_fifo, spawn_fifo_and_convert}; +use super::spawn_fifo; -#[cfg(test)] -mod tests; +// #[cfg(test)] +// mod tests; /// Adjusted batch size for halo2 batches. /// @@ -53,7 +58,7 @@ type BatchVerifier = plonk::BatchVerifier; */ /// The type of verification results. -type VerifyResult = Result<(), Halo2Error>; +type VerifyResult = bool; /// The type of the batch sender channel. type Sender = watch::Sender>; @@ -88,13 +93,13 @@ lazy_static::lazy_static! { /// A Halo2 verification item, used as the request type of the service. #[derive(Clone, Debug)] pub struct Item { - instances: Vec, - proof: orchard::circuit::Proof, + bundle: orchard::bundle::Bundle, + sighash: SigHash, } impl ItemSize for Item { fn item_size(&self) -> usize { - self.instances.len() + self.bundle.actions().len() } } @@ -103,74 +108,84 @@ impl Item { /// /// This is useful (in combination with `Item::clone`) for implementing /// fallback logic when batch verification fails. - pub fn verify_single(&self, vk: &ItemVerifyingKey) -> Result<(), halo2::plonk::Error> { - self.proof.verify(vk, &self.instances[..]) + pub fn verify_single(self, vk: &ItemVerifyingKey) -> bool { + let mut batch = BatchValidator::default(); + batch.queue(self); + batch.validate(vk, thread_rng()) } } -/// A fake batch verifier that queues and verifies halo2 proofs. -#[derive(Default)] -pub struct BatchVerifier { - queue: Vec, +trait QueueBatchVerify { + fn queue(&mut self, item: Item); } -impl BatchVerifier { - /// Queues an item for fake batch verification. - pub fn queue(&mut self, item: Item) { - self.queue.push(item); +impl QueueBatchVerify for BatchValidator { + fn queue(&mut self, Item { bundle, sighash }: Item) { + self.add_bundle(&bundle, sighash.0); } +} - /// Verifies the current fake batch. - pub fn verify( - self, - _rng: R, - vk: &ItemVerifyingKey, - ) -> Result<(), halo2::plonk::Error> { - for item in self.queue { - item.verify_single(vk)?; - } +impl From<(&zebra_chain::orchard::ShieldedData, SigHash)> for Item { + fn from((shielded_data, sighash): (&zebra_chain::orchard::ShieldedData, SigHash)) -> Item { + let anchor = orchard::tree::Anchor::from_bytes(shielded_data.shared_anchor.into()).unwrap(); + let authorization = orchard::bundle::Authorized::from_parts( + orchard::Proof::new(shielded_data.proof.0.clone()), + <[u8; 64]>::from(shielded_data.binding_sig).into(), + ); - Ok(()) - } -} + let bundle = orchard::bundle::Bundle::from_parts( + NonEmpty::from_vec( + shielded_data + .actions + .iter() + .map( + |zebra_chain::orchard::AuthorizedAction { + action: + zebra_chain::orchard::Action { + cv, + nullifier, + rk, + cm_x, + ephemeral_key, + enc_ciphertext, + out_ciphertext, + }, + spend_auth_sig, + }: &zebra_chain::orchard::AuthorizedAction| { + let rk = <[u8; 32]>::from(*rk).try_into().expect("should be valid"); + let cmx = ExtractedNoteCommitment::from_bytes(&<[u8; 32]>::from(*cm_x)) + .expect("should be valid"); + let encrypted_note = TransmittedNoteCiphertext { + epk_bytes: ephemeral_key.into(), + enc_ciphertext: enc_ciphertext.into(), + out_ciphertext: out_ciphertext.into(), + }; + let cv_net = + orchard::value::ValueCommitment::from_bytes(&<[u8; 32]>::from(*cv)) + .expect("should be valid"); + + let spend_auth_sig = <[u8; 64]>::from(*spend_auth_sig).into(); + + orchard::Action::from_parts( + nullifier.into(), + rk, + cmx, + encrypted_note, + cv_net, + spend_auth_sig, + ) + }, + ) + .collect::>(), + ) + .expect("should be valid tx format"), + orchard::bundle::Flags::from_byte(shielded_data.flags.bits()).expect("should be valid"), + shielded_data.value_balance, + anchor, + authorization, + ); -// === END TEMPORARY BATCH HALO2 SUBSTITUTE === - -impl From<&zebra_chain::orchard::ShieldedData> for Item { - fn from(shielded_data: &zebra_chain::orchard::ShieldedData) -> Item { - use orchard::{circuit, note, primitives::redpallas, tree, value}; - - let anchor = tree::Anchor::from_bytes(shielded_data.shared_anchor.into()).unwrap(); - - let enable_spend = shielded_data - .flags - .contains(zebra_chain::orchard::Flags::ENABLE_SPENDS); - let enable_output = shielded_data - .flags - .contains(zebra_chain::orchard::Flags::ENABLE_OUTPUTS); - - let instances = shielded_data - .actions() - .map(|action| { - circuit::Instance::from_parts( - anchor, - value::ValueCommitment::from_bytes(&action.cv.into()).unwrap(), - note::Nullifier::from_bytes(&action.nullifier.into()).unwrap(), - redpallas::VerificationKey::::try_from(<[u8; 32]>::from( - action.rk, - )) - .expect("should be a valid redpallas spendauth verification key"), - note::ExtractedNoteCommitment::from_bytes(&action.cm_x.into()).unwrap(), - enable_spend, - enable_output, - ) - }) - .collect(); - - Item { - instances, - proof: orchard::circuit::Proof::new(shielded_data.proof.0.clone()), - } + Self { bundle, sighash } } } @@ -242,8 +257,8 @@ pub static VERIFIER: Lazy< /// Halo2 verifier. It handles batching incoming requests, driving batches to /// completion, and reporting results. pub struct Verifier { - /// The synchronous Halo2 batch verifier. - batch: BatchVerifier, + /// The synchronous Halo2 batch validator. + batch: BatchValidator, /// The halo2 proof verification key. /// @@ -259,14 +274,14 @@ pub struct Verifier { impl Verifier { fn new(vk: &'static ItemVerifyingKey) -> Self { - let batch = BatchVerifier::default(); + let batch = BatchValidator::default(); let (tx, _) = watch::channel(None); Self { batch, vk, tx } } /// Returns the batch verifier and channel sender from `self`, /// replacing them with a new empty batch. - fn take(&mut self) -> (BatchVerifier, &'static BatchVerifyingKey, Sender) { + fn take(&mut self) -> (BatchValidator, &'static BatchVerifyingKey, Sender) { // Use a new verifier and channel for each batch. let batch = mem::take(&mut self.batch); @@ -278,8 +293,8 @@ impl Verifier { /// Synchronously process the batch, and send the result using the channel sender. /// This function blocks until the batch is completed. - fn verify(batch: BatchVerifier, vk: &'static BatchVerifyingKey, tx: Sender) { - let result = batch.verify(thread_rng(), vk).map_err(Halo2Error::from); + fn verify(batch: BatchValidator, vk: &'static BatchVerifyingKey, tx: Sender) { + let result = batch.validate(vk, thread_rng()); let _ = tx.send(Some(result)); } @@ -296,10 +311,10 @@ impl Verifier { /// Flush the batch using a thread pool, and return the result via the channel. /// This function returns a future that becomes ready when the batch is completed. - async fn flush_spawning(batch: BatchVerifier, vk: &'static BatchVerifyingKey, tx: Sender) { + async fn flush_spawning(batch: BatchValidator, vk: &'static BatchVerifyingKey, tx: Sender) { // Correctness: Do CPU-intensive work on a dedicated thread, to avoid blocking other futures. let _ = tx.send( - spawn_fifo(move || batch.verify(thread_rng(), vk).map_err(Halo2Error::from)) + spawn_fifo(move || batch.validate(vk, thread_rng())) .await .ok(), ); @@ -310,8 +325,13 @@ impl Verifier { item: Item, pvk: &'static ItemVerifyingKey, ) -> Result<(), BoxError> { + // TODO: Restore code for verifying single proofs or return a result from batch.validate() // Correctness: Do CPU-intensive work on a dedicated thread, to avoid blocking other futures. - spawn_fifo_and_convert(move || item.verify_single(pvk).map_err(Halo2Error::from)).await + if spawn_fifo(move || item.verify_single(pvk)).await? { + Ok(()) + } else { + Err("could not validate orchard proof".into()) + } } } @@ -346,21 +366,20 @@ impl Service> for Verifier { Ok(()) => { // We use a new channel for each batch, // so we always get the correct batch result here. - let result = rx + let is_valid = *rx .borrow() .as_ref() - .ok_or("threadpool unexpectedly dropped response channel sender. Is Zebra shutting down?")? - .clone(); + .ok_or("threadpool unexpectedly dropped response channel sender. Is Zebra shutting down?")?; - if result.is_ok() { - tracing::trace!(?result, "verified halo2 proof"); + if is_valid { + tracing::trace!(?is_valid, "verified halo2 proof"); metrics::counter!("proofs.halo2.verified").increment(1); + Ok(()) } else { - tracing::trace!(?result, "invalid halo2 proof"); + tracing::trace!(?is_valid, "invalid halo2 proof"); metrics::counter!("proofs.halo2.invalid").increment(1); + Err("could not validate halo2 proofs".into()) } - - result.map_err(BoxError::from) } Err(_recv_error) => panic!("verifier was dropped without flushing"), } diff --git a/zebra-consensus/src/primitives/halo2/tests.rs b/zebra-consensus/src/primitives/halo2/tests.rs index e654adcc546..6054767a3bf 100644 --- a/zebra-consensus/src/primitives/halo2/tests.rs +++ b/zebra-consensus/src/primitives/halo2/tests.rs @@ -154,8 +154,13 @@ async fn verify_generated_halo2_proofs() { crate::primitives::MAX_BATCH_LATENCY, ), tower::service_fn( - (|item: Item| ready(item.verify_single(&VERIFYING_KEY).map_err(Halo2Error::from))) - as fn(_) -> _, + (|item: Item| { + ready( + item.verify_single(&VERIFYING_KEY) + .then_some(()) + .ok_or("could not validate orchard proof"), + ) + }) as fn(_) -> _, ), ); @@ -221,8 +226,13 @@ async fn correctly_err_on_invalid_halo2_proofs() { crate::primitives::MAX_BATCH_LATENCY, ), tower::service_fn( - (|item: Item| ready(item.verify_single(&VERIFYING_KEY).map_err(Halo2Error::from))) - as fn(_) -> _, + (|item: Item| { + ready( + item.verify_single(&VERIFYING_KEY) + .then_some(()) + .ok_or("could not validate orchard proof"), + ) + }) as fn(_) -> _, ), ); diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 661b0be14d2..a109b695a98 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -1275,81 +1275,8 @@ where // aggregated Halo2 proof per transaction, even with multiple // Actions in one transaction. So we queue it for verification // only once instead of queuing it up for every Action description. - async_checks.push( - primitives::halo2::VERIFIER - .clone() - .oneshot(primitives::halo2::Item::from(orchard_shielded_data)), - ); - - for authorized_action in orchard_shielded_data.actions.iter().cloned() { - let (action, spend_auth_sig) = authorized_action.into_parts(); - - // # Consensus - // - // > - Let SigHash be the SIGHASH transaction hash of this transaction, not - // > associated with an input, as defined in § 4.10 using SIGHASH_ALL. - // > - The spend authorization signature MUST be a valid SpendAuthSig^{Orchard} - // > signature over SigHash using rk as the validating key — i.e. - // > SpendAuthSig^{Orchard}.Validate_{rk}(SigHash, spendAuthSig) = 1. - // > As specified in § 5.4.7, validation of the 𝑅 component of the - // > signature prohibits non-canonical encodings. - // - // https://zips.z.cash/protocol/protocol.pdf#actiondesc - // - // This is validated by the verifier, inside the [`reddsa`] crate. - // It calls [`pallas::Affine::from_bytes`] to parse R and - // that enforces the canonical encoding. - // - // Queue the validation of the RedPallas spend - // authorization signature for each Action - // description while adding the resulting future to - // our collection of async checks that (at a - // minimum) must pass for the transaction to verify. - async_checks.push(primitives::redpallas::VERIFIER.clone().oneshot( - primitives::redpallas::Item::from_spendauth( - action.rk, - spend_auth_sig, - &shielded_sighash, - ), - )); - } - - let bvk = orchard_shielded_data.binding_verification_key(); - - // # Consensus - // - // > The Action transfers of a transaction MUST be consistent with - // > its v balanceOrchard value as specified in § 4.14. - // - // https://zips.z.cash/protocol/protocol.pdf#actions - // - // > [NU5 onward] If effectiveVersion ≥ 5 and nActionsOrchard > 0, then: - // > – let bvk^{Orchard} and SigHash be as defined in § 4.14; - // > – bindingSigOrchard MUST represent a valid signature under the - // > transaction binding validating key bvk^{Orchard} of SigHash — - // > i.e. BindingSig^{Orchard}.Validate_{bvk^{Orchard}}(SigHash, bindingSigOrchard) = 1. - // - // https://zips.z.cash/protocol/protocol.pdf#txnconsensus - // - // This is validated by the verifier. The `if` part is indirectly - // enforced, since the `orchard_shielded_data` is only parsed if those - // conditions apply in [`Transaction::zcash_deserialize`]. - // - // > As specified in § 5.4.7, validation of the 𝑅 component of the signature - // > prohibits non-canonical encodings. - // - // https://zips.z.cash/protocol/protocol.pdf#txnconsensus - // - // This is validated by the verifier, inside the `reddsa` crate. - // It calls [`pallas::Affine::from_bytes`] to parse R and - // that enforces the canonical encoding. - - async_checks.push(primitives::redpallas::VERIFIER.clone().oneshot( - primitives::redpallas::Item::from_binding( - bvk, - orchard_shielded_data.binding_sig, - &shielded_sighash, - ), + async_checks.push(primitives::halo2::VERIFIER.clone().oneshot( + primitives::halo2::Item::from((orchard_shielded_data, *shielded_sighash)), )); } From 89a9f1a5159a77659fc961a89bc2fdc1f6ea2bb3 Mon Sep 17 00:00:00 2001 From: Arya Date: Fri, 28 Feb 2025 01:20:35 -0500 Subject: [PATCH 4/4] Increments `pending_items` on tower-batch-control `Workers` by `item_size()` instead of 1, and increases halo2 max batch size from 2 to 64 --- tower-batch-control/src/worker.rs | 3 +- zebra-consensus/src/primitives/halo2.rs | 38 ++----------------------- 2 files changed, 4 insertions(+), 37 deletions(-) diff --git a/tower-batch-control/src/worker.rs b/tower-batch-control/src/worker.rs index 7bf8fd55786..3c96f12b745 100644 --- a/tower-batch-control/src/worker.rs +++ b/tower-batch-control/src/worker.rs @@ -144,10 +144,9 @@ where match self.service.ready().await { Ok(svc) => { + self.pending_items += req.item_size(); let rsp = svc.call(req.into()); let _ = tx.send(Ok(rsp)); - - self.pending_items += 1; } Err(e) => { self.failed(e.into()); diff --git a/zebra-consensus/src/primitives/halo2.rs b/zebra-consensus/src/primitives/halo2.rs index 7f59c16d554..b9e4d5ac0da 100644 --- a/zebra-consensus/src/primitives/halo2.rs +++ b/zebra-consensus/src/primitives/halo2.rs @@ -28,34 +28,15 @@ use tower_fallback::Fallback; use super::spawn_fifo; -// #[cfg(test)] -// mod tests; - /// Adjusted batch size for halo2 batches. /// /// Unlike other batch verifiers, halo2 has aggregate proofs. /// This means that there can be hundreds of actions verified by some proofs, /// but just one action in others. /// -/// To compensate for larger proofs, we decrease the batch size. -/// -/// We also decrease the batch size for these reasons: -/// - the default number of actions in `zcashd` is 2, -/// - halo2 proofs take longer to verify than Sapling proofs, and -/// - transactions with many actions generate very large proofs. -/// -/// # TODO -/// -/// Count each halo2 action as a batch item. -/// We could increase the batch item count by the action count each time a batch request -/// is received, which would reduce batch size, but keep the batch queue size larger. -const HALO2_MAX_BATCH_SIZE: usize = 2; - -/* TODO: implement batch verification - -/// The type of the batch verifier. -type BatchVerifier = plonk::BatchVerifier; - */ +/// To compensate for larger proofs, we process the batch once there are over +/// [`HALO2_MAX_BATCH_SIZE`] total actions among pending items in the queue. +const HALO2_MAX_BATCH_SIZE: usize = super::MAX_BATCH_SIZE; /// The type of verification results. type VerifyResult = bool; @@ -63,12 +44,6 @@ type VerifyResult = bool; /// The type of the batch sender channel. type Sender = watch::Sender>; -/* TODO: implement batch verification - -/// The type of a raw verifying key. -/// This is the key used to verify batches. -pub type BatchVerifyingKey = VerifyingKey; - */ /// Temporary substitute type for fake batch verification. /// /// TODO: implement batch verification @@ -83,13 +58,6 @@ lazy_static::lazy_static! { pub static ref VERIFYING_KEY: ItemVerifyingKey = ItemVerifyingKey::build(); } -// === TEMPORARY BATCH HALO2 SUBSTITUTE === -// -// These types are meant to be API compatible with the batch verification APIs -// in bellman::groth16::batch, reddsa::batch, redjubjub::batch, and -// ed25519-zebra::batch. Once Halo2 batch proof verification math and -// implementation is available, this code can be replaced with that. - /// A Halo2 verification item, used as the request type of the service. #[derive(Clone, Debug)] pub struct Item {