feat: merge-train/barretenberg (#18107)

BEGIN_COMMIT_OVERRIDE
Improve pairing point tagging testing
chore: audit ram/rom in stdlib (#18023)
fix: Add AVM build to ci-barretenberg (#18109)
chore: merge merge verifiers (#18086)
feat(bb): upload benchmark breakdowns to CI disk storage (#17871)
fix: biggroup `batch_mul` fixes and cleanup (#17903)
fix: disable AVM in bb merge train again (#18118) I
chore: Add ecdsa batch mul documentation (#17983)
chore: Ec operations audit, part 2 (#18088)
END_COMMIT_OVERRIDE

Author: AztecBot

barretenberg/cpp/scripts/ci_benchmark_ivc_flows.sh

@@ -2,6 +2,8 @@
 # Performs the chonk private transaction proving benchmarks for our 'realistic apps'.
 # This is called by yarn-project/end-to-end/bootstrap.sh bench, which creates these inputs from end-to-end tests.
 source $(git rev-parse --show-toplevel)/ci3/source
+source $(git rev-parse --show-toplevel)/ci3/source_redis
+source $(git rev-parse --show-toplevel)/ci3/source_cache
 
 if [[ $# -ne 2 ]]; then
   echo "Usage: $0 <runtime> <benchmark_folder>"
@@ -45,15 +47,16 @@ function run_bb_cli_bench {
   shift 2
 
   if [[ "$runtime" == "native" ]]; then
-    memusage "./$native_build_dir/bin/bb" "$@" || {
-      echo "bb native failed with args: $@"
+    # Add --bench_out_hierarchical flag for native builds to capture hierarchical op counts and timings
+    memusage "./$native_build_dir/bin/bb" "$@" "--bench_out_hierarchical" "$output/benchmark_breakdown.json" || {
+      echo "bb native failed with args: $@ --bench_out_hierarchical $output/benchmark_breakdown.json"
       exit 1
     }
   else # wasm
     export WASMTIME_ALLOWED_DIRS="--dir=$flow_folder --dir=$output"
-    # TODO support wasm op count time preset
-    memusage scripts/wasmtime.sh $WASMTIME_ALLOWED_DIRS ./build-wasm-threads/bin/bb "$@" || {
-      echo "bb wasm failed with args: $@"
+    # Add --bench_out_hierarchical flag for wasm builds to capture hierarchical op counts and timings
+    memusage scripts/wasmtime.sh $WASMTIME_ALLOWED_DIRS ./build-wasm-threads/bin/bb "$@" "--bench_out_hierarchical" "$output/benchmark_breakdown.json" || {
+      echo "bb wasm failed with args: $@ --bench_out_hierarchical $output/benchmark_breakdown.json"
       exit 1
     }
   fi
@@ -102,3 +105,37 @@ EOF
 export -f verify_ivc_flow run_bb_cli_bench
 
 chonk_flow $1 $2
+
+# Upload benchmark breakdown (op counts and timings) to disk if running in CI
+if [[ "${CI:-}" == "1" ]] && [[ "${CI_ENABLE_DISK_LOGS:-0}" == "1" ]]; then
+  echo_header "Uploading Barretenberg benchmark breakdowns"
+
+  runtime="$1"
+  flow_name="$(basename $2)"
+  benchmark_breakdown_file="bench-out/app-proving/$flow_name/$runtime/benchmark_breakdown.json"
+
+  if [[ -f "$benchmark_breakdown_file" ]]; then
+    current_sha=$(git rev-parse HEAD)
+
+    # Create cache key: bench-bb-breakdown-<runtime>-<flow_name>-<sha>
+    # This will be accessible at: http://ci.aztec-labs.com/bench-bb-breakdown-<runtime>-<flow_name>-<sha>
+    cache_key="bench-bb-breakdown-${runtime}-${flow_name}-${current_sha}"
+
+    # Upload to Redis (30 day retention) and disk (bench/bb-breakdown subfolder)
+    {
+      # Write to Redis for ci.aztec-labs.com access
+      cat "$benchmark_breakdown_file" | gzip | redis_cli -x SETEX "$cache_key" 2592000 &>/dev/null
+
+      # Write to disk in explicit subfolder (only if disk logging enabled)
+      if [[ "${CI_ENABLE_DISK_LOGS:-0}" == "1" ]]; then
+        # Strip the prefix from key when writing to disk subfolder
+        disk_key="${cache_key#bench-bb-breakdown-}"
+        cat "$benchmark_breakdown_file" | gzip | cache_disk_transfer_to "bench/bb-breakdown" "$disk_key"
+      fi
+    } &
+
+    echo "Uploaded benchmark breakdown: http://ci.aztec-labs.com/$cache_key"
+  else
+    echo "Warning: benchmark breakdown file not found at $benchmark_breakdown_file"
+  fi
+fi

barretenberg/cpp/scripts/test_chonk_standalone_vks_havent_changed.sh

@@ -13,7 +13,7 @@ cd ..
 # - Generate a hash for versioning: sha256sum bb-chonk-inputs.tar.gz
 # - Upload the compressed results: aws s3 cp bb-chonk-inputs.tar.gz s3://aztec-ci-artifacts/protocol/bb-chonk-inputs-[hash(0:8)].tar.gz
 # Note: In case of the "Test suite failed to run ... Unexpected token 'with' " error, need to run: docker pull aztecprotocol/build:3.0
-pinned_short_hash="07b5558d"
+pinned_short_hash="fde8277f"
 pinned_chonk_inputs_url="https://aztec-ci-artifacts.s3.us-east-2.amazonaws.com/protocol/bb-chonk-inputs-${pinned_short_hash}.tar.gz"
 
 function compress_and_upload {

barretenberg/cpp/src/CMakeLists.txt

@@ -113,7 +113,8 @@ add_subdirectory(barretenberg/wasi)
 add_subdirectory(barretenberg/world_state)
 add_subdirectory(barretenberg/lmdblib)
 
-if(AVM)
+# NOTE: Do not conditionally base this on the AVM flag as it defines a necessary vm2_sim library.
+if(NOT WASM)
     add_subdirectory(barretenberg/vm2)
     if(FUZZING)
         add_subdirectory(barretenberg/avm_fuzzer)
@@ -176,7 +177,6 @@ set(BARRETENBERG_TARGET_OBJECTS
     $<TARGET_OBJECTS:stdlib_circuit_builders_objects>
     $<TARGET_OBJECTS:stdlib_chonk_verifier_objects>
     $<TARGET_OBJECTS:stdlib_eccvm_verifier_objects>
-    $<TARGET_OBJECTS:stdlib_merge_verifier_objects>
     $<TARGET_OBJECTS:stdlib_honk_verifier_objects>
     $<TARGET_OBJECTS:stdlib_goblin_verifier_objects>
     $<TARGET_OBJECTS:stdlib_keccak_objects>

barretenberg/cpp/src/barretenberg/boomerang_value_detection/CMakeLists.txt

@@ -1,5 +1,5 @@
 barretenberg_module(boomerang_value_detection stdlib_circuit_builders circuit_checker
                     stdlib_primitives numeric stdlib_aes128 stdlib_sha256 stdlib_blake2s
-                    stdlib_blake3s stdlib_poseidon2 stdlib_honk_verifier stdlib_merge_verifier
+                    stdlib_blake3s stdlib_poseidon2 stdlib_honk_verifier
                     stdlib_goblin_verifier stdlib_pedersen_hash goblin
                     commitment_schemes)

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_dynamic_array.test.cpp

@@ -3,7 +3,6 @@
 #include "barretenberg/common/test.hpp"
 #include "barretenberg/ecc/fields/field_conversion.hpp"
 #include "barretenberg/goblin/mock_circuits.hpp"
-#include "barretenberg/stdlib/merge_verifier/merge_recursive_verifier.hpp"
 #include "barretenberg/stdlib/primitives/curves/bn254.hpp"
 #include "barretenberg/ultra_honk/merge_prover.hpp"
 #include "barretenberg/ultra_honk/merge_verifier.hpp"
@@ -24,9 +23,9 @@ namespace bb::stdlib::recursion::goblin {
 template <class RecursiveBuilder> class BoomerangRecursiveMergeVerifierTest : public testing::Test {
 
     // Types for recursive verifier circuit
-    using RecursiveMergeVerifier = MergeRecursiveVerifier_<RecursiveBuilder>;
-    using RecursiveTableCommitments = MergeRecursiveVerifier_<RecursiveBuilder>::TableCommitments;
-    using RecursiveMergeCommitments = MergeRecursiveVerifier_<RecursiveBuilder>::InputCommitments;
+    using RecursiveMergeVerifier = MergeRecursiveVerifier<RecursiveBuilder>;
+    using RecursiveTableCommitments = MergeRecursiveVerifier<RecursiveBuilder>::TableCommitments;
+    using RecursiveMergeCommitments = MergeRecursiveVerifier<RecursiveBuilder>::InputCommitments;
 
     // Define types relevant for inner circuit
     using InnerFlavor = MegaFlavor;
@@ -89,10 +88,10 @@ template <class RecursiveBuilder> class BoomerangRecursiveMergeVerifierTest : pu
         }
 
         // Create a recursive merge verification circuit for the merge proof
-        RecursiveMergeVerifier verifier{ &outer_circuit, settings };
-        verifier.transcript->enable_manifest();
+        auto merge_transcript = std::make_shared<StdlibTranscript<RecursiveBuilder>>();
+        RecursiveMergeVerifier verifier{ settings, merge_transcript };
         const stdlib::Proof<RecursiveBuilder> stdlib_merge_proof(outer_circuit, merge_proof);
-        [[maybe_unused]] auto [pairing_points, recursive_merged_table_commitments] =
+        [[maybe_unused]] auto [pairing_points, recursive_merged_table_commitments, degree_check_verified] =
             verifier.verify_proof(stdlib_merge_proof, recursive_merge_commitments);
 
         // Check for a failure flag in the recursive verifier circuit

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_merge_recursive_verifier.test.cpp

@@ -45,7 +45,7 @@ template <typename RecursiveFlavor> class BoomerangRecursiveVerifierTest : publi
     using RecursiveVerifier = UltraRecursiveVerifier_<RecursiveFlavor>;
     using VerificationKey = typename RecursiveVerifier::VerificationKey;
 
-    using PairingObject = PairingPoints<OuterBuilder>;
+    using PairingObject = PairingPoints<bn254<OuterBuilder>>;
     using VerifierOutput = bb::stdlib::recursion::honk::UltraRecursiveVerifierOutput<OuterBuilder>;
     using StdlibProof = bb::stdlib::Proof<OuterBuilder>;
 

barretenberg/cpp/src/barretenberg/boomerang_value_detection/graph_description_ultra_recursive_verifier.test.cpp

@@ -60,7 +60,7 @@ class Chonk : public IVCBase {
     using RecursiveVerificationKey = RecursiveFlavor::VerificationKey;
     using RecursiveVKAndHash = RecursiveFlavor::VKAndHash;
     using RecursiveTranscript = RecursiveFlavor::Transcript;
-    using PairingPoints = stdlib::recursion::PairingPoints<ClientCircuit>;
+    using PairingPoints = stdlib::recursion::PairingPoints<stdlib::bn254<ClientCircuit>>;
     using KernelIO = bb::stdlib::recursion::honk::KernelIO;
     using HidingKernelIO = bb::stdlib::recursion::honk::HidingKernelIO<ClientCircuit>;
     using AppIO = bb::stdlib::recursion::honk::AppIO;

barretenberg/cpp/src/barretenberg/chonk/chonk.hpp

@@ -17,11 +17,10 @@ class CircuitChecker {
     template <typename Builder> static bool check(const Builder& builder)
     {
         static_assert(IsCheckable<Builder>);
-        if constexpr (IsUltraBuilder<Builder> || IsMegaBuilder<Builder>) {
+        if (IsUltraOrMegaBuilder<Builder>) {
             return UltraCircuitChecker::check(builder);
-        } else {
-            return false;
         }
+        return false;
     }
 };
 

barretenberg/cpp/src/barretenberg/circuit_checker/circuit_checker.hpp

@@ -8,8 +8,10 @@
 
 #include "barretenberg/commitment_schemes/claim.hpp"
 #include "barretenberg/commitment_schemes/commitment_key.hpp"
+#include "barretenberg/commitment_schemes/pairing_points.hpp"
 #include "barretenberg/commitment_schemes/utils/batch_mul_native.hpp"
 #include "barretenberg/commitment_schemes/verification_key.hpp"
+#include "barretenberg/stdlib/primitives/pairing_points.hpp"
 #include "barretenberg/transcript/transcript.hpp"
 
 #include <memory>
@@ -26,7 +28,8 @@ template <typename Curve_> class KZG {
     using Commitment = typename Curve::AffineElement;
     using GroupElement = typename Curve::Element;
     using Polynomial = bb::Polynomial<Fr>;
-    using VerifierAccumulator = std::array<GroupElement, 2>;
+    using PairingPointsType =
+        std::conditional_t<Curve::is_stdlib_type, stdlib::recursion::PairingPoints<Curve>, bb::PairingPoints<Curve>>;
 
     /**
      * @brief Computes the KZG commitment to an opening proof polynomial at a single evaluation point
@@ -59,6 +62,10 @@ template <typename Curve_> class KZG {
         // future we might need to adjust this to use the incoming alternative to work queue (i.e. variation of
         // pthreads) or even the work queue itself
         prover_trancript->send_to_verifier("KZG:W", quotient_commitment);
+
+        // Masking challenge is used in the recursive setting to perform batch_mul. This is not used
+        // by the prover directly, we just need it for consistent transcript state with the verifier.
+        prover_trancript->template get_challenge<Fr>("KZG:masking_challenge");
     };
 
     /**
@@ -67,13 +74,13 @@ template <typename Curve_> class KZG {
      * @details This is used in the recursive setting where we want to "aggregate" proofs, not verify them.
      *
      * @param claim OpeningClaim ({r, v}, C)
-     * @return  {P₀, P₁} where
+     * @return  PairingPoints {P₀, P₁} where
      *      - P₀ = C − v⋅[1]₁ + r⋅[W(x)]₁
      *      - P₁ = - [W(x)]₁
      */
     template <typename Transcript>
-    static VerifierAccumulator reduce_verify(const OpeningClaim<Curve>& claim,
-                                             const std::shared_ptr<Transcript>& verifier_transcript)
+    static PairingPointsType reduce_verify(const OpeningClaim<Curve>& claim,
+                                           const std::shared_ptr<Transcript>& verifier_transcript)
     {
         auto quotient_commitment = verifier_transcript->template receive_from_prover<Commitment>("KZG:W");
 
@@ -98,7 +105,7 @@ template <typename Curve_> class KZG {
         }
 
         auto P_1 = -quotient_commitment;
-        return { P_0, P_1 };
+        return PairingPointsType(P_0, P_1);
     };
 
     /**
@@ -114,16 +121,19 @@ template <typename Curve_> class KZG {
      *
      * @param batch_opening_claim \f$(\text{commitments}, \text{scalars}, \text{shplonk_evaluation_challenge})\f$
      *        A struct containing the commitments, scalars, and the Shplonk evaluation challenge.
-     * @return \f$ \{P_0, P_1\}\f$ where:
+     * @return PairingPoints \f$ \{P_0, P_1\}\f$ where:
      *         - \f$ P_0 = C + [W(x)]_1 \cdot z \f$
      *         - \f$ P_1 = - [W(x)]_1 \f$
      */
     template <typename Transcript>
-    static VerifierAccumulator reduce_verify_batch_opening_claim(BatchOpeningClaim<Curve> batch_opening_claim,
-                                                                 const std::shared_ptr<Transcript>& transcript)
+    static PairingPointsType reduce_verify_batch_opening_claim(BatchOpeningClaim<Curve> batch_opening_claim,
+                                                               const std::shared_ptr<Transcript>& transcript)
     {
         auto quotient_commitment = transcript->template receive_from_prover<Commitment>("KZG:W");
 
+        // This challenge is used to compute offset generators in the batch_mul call below
+        const Fr masking_challenge = transcript->template get_challenge<Fr>("KZG:masking_challenge");
+
         // The pairing check can be expressed as
         // e(C + [W]₁ ⋅ z, [1]₂) * e(−[W]₁, [X]₂) = 1, where C = ∑ commitmentsᵢ ⋅ scalarsᵢ.
         GroupElement P_0;
@@ -136,13 +146,15 @@ template <typename Curve_> class KZG {
             P_0 = GroupElement::batch_mul(batch_opening_claim.commitments,
                                           batch_opening_claim.scalars,
                                           /*max_num_bits=*/0,
-                                          /*with_edgecases=*/true);
+                                          /*with_edgecases=*/true,
+                                          /*masking_scalar=*/masking_challenge);
         } else {
             P_0 = batch_mul_native(batch_opening_claim.commitments, batch_opening_claim.scalars);
         }
         auto P_1 = -quotient_commitment;
 
-        return { P_0, P_1 };
+        return PairingPointsType(P_0, P_1);
     }
 };
+
 } // namespace bb