Merge branch 'next' into merge-train/barretenberg

Author: AztecBot

.test_patterns.yml

@@ -291,8 +291,3 @@ tests:
     flake_group_id: e2e-p2p-epoch-flakes
     owners:
       - *palla
-
-  - regex: "bb-prover/src/avm_proving_tests/avm_bulk.test.ts"
-    error_regex: "Exceeded timeout"
-    owners:
-      - *alex

barretenberg/cpp/CMakeLists.txt

@@ -87,8 +87,6 @@ if(FUZZING)
     if(DISABLE_CUSTOM_MUTATORS)
         add_definitions(-DDISABLE_CUSTOM_MUTATORS=1)
     endif()
-
-    set(MULTITHREADING OFF)
 endif()
 
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "wasm32")

barretenberg/cpp/CMakePresets.json

@@ -273,6 +273,20 @@
         "CXXFLAGS": "-fprofile-instr-generate -fcoverage-mapping"
       }
     },
+    {
+      "name": "fuzzing-avm",
+      "displayName": "Build with fuzzing and AVM",
+      "description": "Build default preset but with fuzzing and AVM enabled",
+      "inherits": "clang20",
+      "binaryDir": "build-fuzzing-avm",
+      "cacheVariables": {
+        "FUZZING": "ON",
+        "DISABLE_ASM": "OFF",
+        "AVM_TRANSPILER_LIB": "",
+        "CMAKE_BUILD_TYPE": "RelWithAssert",
+        "MULTITHREADING": "ON"
+      }
+    },
     {
       "name": "smt-verification",
       "displayName": "Build with smt verification",
@@ -580,6 +594,11 @@
       "inherits": "clang20",
       "configurePreset": "fuzzing-coverage"
     },
+    {
+      "name": "fuzzing-avm",
+      "inherits": "clang20",
+      "configurePreset": "fuzzing-avm"
+    },
     {
       "name": "gperftools",
       "inherits": "clang20",

barretenberg/cpp/src/CMakeLists.txt

@@ -110,13 +110,19 @@ add_subdirectory(barretenberg/translator_vm)
 add_subdirectory(barretenberg/ultra_honk)
 add_subdirectory(barretenberg/vm2_stub)
 add_subdirectory(barretenberg/wasi)
+add_subdirectory(barretenberg/world_state)
+add_subdirectory(barretenberg/lmdblib)
+
+if(AVM)
+    add_subdirectory(barretenberg/vm2)
+    if(FUZZING)
+        add_subdirectory(barretenberg/avm_fuzzer)
+    endif()
+endif()
 
 if(NOT FUZZING AND NOT WASM)
     add_subdirectory(barretenberg/ipc)
-    add_subdirectory(barretenberg/lmdblib)
     add_subdirectory(barretenberg/nodejs_module)
-    add_subdirectory(barretenberg/world_state)
-    add_subdirectory(barretenberg/vm2)
 endif()
 
 if(SMT)

barretenberg/cpp/src/barretenberg/avm_fuzzer/CMakeLists.txt

@@ -0,0 +1,4 @@
+if(AVM AND FUZZING)
+    barretenberg_module(avm_fuzzer vm2 nlohmann_json::nlohmann_json)
+endif()
+

barretenberg/cpp/src/barretenberg/avm_fuzzer/avm.fuzzer.cpp

@@ -0,0 +1,75 @@
+#include <iomanip>
+#include <iostream>
+#include <random>
+#include <string>
+#include <vector>
+
+#include "barretenberg/avm_fuzzer/fuzz_lib/control_flow.hpp"
+#include "barretenberg/avm_fuzzer/fuzz_lib/fuzz.hpp"
+#include "barretenberg/avm_fuzzer/fuzz_lib/fuzzer_data.hpp"
+#include "barretenberg/avm_fuzzer/fuzz_lib/simulator.hpp"
+#include "barretenberg/avm_fuzzer/mutations/fuzzer_data.hpp"
+#include "barretenberg/serialize/msgpack_impl.hpp"
+
+using FuzzInstruction = ::FuzzInstruction;
+
+/// Initializes the typescript simulator process
+/// See yarn-project/simulator/scripts/fuzzing/
+extern "C" int LLVMFuzzerInitialize(int*, char***)
+{
+
+    const char* simulator_path = std::getenv("AVM_SIMULATOR_BIN");
+    if (simulator_path == nullptr) {
+        throw std::runtime_error("AVM_SIMULATOR_BIN is not set");
+    }
+    std::string simulator_path_str(simulator_path);
+    JsSimulator::initialize(simulator_path_str);
+    return 0;
+}
+
+SimulatorResult fuzz(const uint8_t* buffer, size_t size)
+{
+    FuzzerData deserialized_data;
+    try {
+        msgpack::unpack((reinterpret_cast<const char*>(buffer)), size).get().convert(deserialized_data);
+    } catch (const std::exception& e) {
+        deserialized_data = FuzzerData();
+    }
+    auto res = fuzz(deserialized_data);
+
+    return res;
+}
+
+extern "C" size_t LLVMFuzzerCustomMutator(uint8_t* serialized_fuzzer_data,
+                                          size_t serialized_fuzzer_data_size,
+                                          size_t max_size,
+                                          unsigned int seed)
+{
+    auto rng = std::mt19937_64(seed);
+    FuzzerData deserialized_data;
+    try {
+        msgpack::unpack((reinterpret_cast<const char*>(serialized_fuzzer_data)), serialized_fuzzer_data_size)
+            .get()
+            .convert(deserialized_data);
+    } catch (const std::exception& e) {
+        deserialized_data = FuzzerData();
+    }
+    mutate_fuzzer_data(deserialized_data, rng);
+    auto [mutated_serialized_fuzzer_data, mutated_serialized_fuzzer_data_size] =
+        msgpack_encode_buffer(deserialized_data);
+    if (mutated_serialized_fuzzer_data_size > max_size) {
+        delete[] mutated_serialized_fuzzer_data;
+        return 0;
+    }
+
+    memcpy(serialized_fuzzer_data, mutated_serialized_fuzzer_data, mutated_serialized_fuzzer_data_size);
+    delete[] mutated_serialized_fuzzer_data;
+
+    return mutated_serialized_fuzzer_data_size;
+}
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
+{
+    fuzz(data, size);
+    return 0;
+}

barretenberg/cpp/src/barretenberg/avm_fuzzer/common/process.cpp

@@ -0,0 +1,66 @@
+#include "process.hpp"
+
+Process::Process(const std::string& command)
+{
+    int stdin_pipe[2];  // NOLINT
+    int stdout_pipe[2]; // NOLINT
+    pid_t pid = 0;
+
+    if (pipe(stdin_pipe) < 0 || pipe(stdout_pipe) < 0) {
+        throw std::runtime_error("pipe() failed");
+    }
+
+    pid = fork();
+    if (pid < 0) {
+        throw std::runtime_error("fork() failed");
+    }
+
+    if (pid == 0) { // Child process
+        close(stdin_pipe[1]);
+        close(stdout_pipe[0]);
+
+        dup2(stdin_pipe[0], STDIN_FILENO);
+        dup2(stdout_pipe[1], STDOUT_FILENO);
+
+        close(stdin_pipe[0]);
+        close(stdout_pipe[1]);
+
+        execl("/bin/sh", "sh", "-c", command.c_str(), nullptr); // NOLINT
+        exit(1);
+    }
+
+    // Parent process
+    close(stdin_pipe[0]);
+    close(stdout_pipe[1]);
+
+    this->pid = pid;
+    this->stdin_fd = stdin_pipe[1];
+    this->stdout_fd = stdout_pipe[0];
+}
+
+Process::~Process()
+{
+    close(stdin_fd);
+    close(stdout_fd);
+    waitpid(pid, nullptr, 0);
+}
+
+void Process::write_line(const std::string& line) const
+{
+    std::string command = line + "\n";
+    write(stdin_fd, command.c_str(), command.size());
+}
+
+std::string Process::read_line() const
+{
+    char buffer[4096]; // NOLINT
+    std::string response;
+    ssize_t bytes_read = 0;
+    while ((bytes_read = read(stdout_fd, buffer, sizeof(buffer))) > 0) {
+        response.append(buffer, static_cast<size_t>(bytes_read));
+        if (response.find('\n') != std::string::npos) {
+            break;
+        }
+    }
+    return response;
+}

barretenberg/cpp/src/barretenberg/avm_fuzzer/common/process.hpp

@@ -0,0 +1,27 @@
+#include <array>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <sys/wait.h>
+#include <unistd.h>
+
+class Process {
+  private:
+    pid_t pid;
+    int stdin_fd = 0;
+    int stdout_fd = 0;
+
+  public:
+    Process(const std::string& command);
+    ~Process();
+    Process(const Process&) = delete;
+    Process& operator=(const Process&) = delete;
+    Process(Process&&) = default;
+    Process& operator=(Process&&) = default;
+
+    /// Ends line with a newline character, sends to the process.
+    void write_line(const std::string& line) const;
+
+    /// Reads a line from the process.
+    std::string read_line() const;
+};

barretenberg/cpp/src/barretenberg/avm_fuzzer/common/weighted_selection.hpp

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <array>
+#include <random>
+#include <stdexcept>
+#include <vector>
+
+template <typename T, size_t N> class WeightedSelectionConfig {
+  private:
+    std::array<std::pair<T, size_t>, N> options_with_weights;
+    size_t total_weight = 0;
+
+  public:
+    constexpr WeightedSelectionConfig(const std::array<std::pair<T, size_t>, N>& options_with_weights)
+        : options_with_weights(options_with_weights)
+    {
+        for (const auto& [option, weight] : options_with_weights) {
+            total_weight += weight;
+        }
+    }
+
+    constexpr WeightedSelectionConfig(std::initializer_list<std::pair<T, size_t>> options_with_weights)
+        : options_with_weights()
+    {
+        size_t i = 0;
+        for (const auto& [option, weight] : options_with_weights) {
+            this->options_with_weights[i] = { option, weight };
+            total_weight += weight;
+            ++i;
+        }
+    }
+
+    T select(std::mt19937_64& rng) const
+    {
+        std::uniform_int_distribution<size_t> dist(0, total_weight - 1);
+        size_t selector = dist(rng);
+
+        for (const auto& [option, weight] : options_with_weights) {
+            if (selector < weight) {
+                return option;
+            }
+            selector -= weight;
+        }
+
+        throw std::runtime_error("Should have returned by now");
+    }
+};

barretenberg/cpp/src/barretenberg/avm_fuzzer/fuzz_lib/control_flow.cpp

@@ -0,0 +1,40 @@
+#include "barretenberg/avm_fuzzer/fuzz_lib/control_flow.hpp"
+
+void ControlFlow::process_insert_simple_instruction_block(InsertSimpleInstructionBlock instruction)
+{
+    if (instruction_blocks->size() == 0) {
+        return;
+    }
+    auto instruction_block = instruction_blocks->at(instruction.instruction_block_idx % instruction_blocks->size());
+    for (const auto& instr : instruction_block) {
+        current_block.process_instruction(instr);
+    }
+}
+
+void ControlFlow::process_cfg_instruction(CFGInstruction instruction)
+{
+    std::visit(overloaded_cfg_instruction{ [&](InsertSimpleInstructionBlock arg) {
+                   process_insert_simple_instruction_block(arg);
+               } },
+               instruction);
+}
+
+// Helper function to create bytecode from a vector of instructions
+std::vector<uint8_t> create_bytecode(const std::vector<bb::avm2::simulation::Instruction>& instructions)
+{
+    std::vector<uint8_t> bytecode;
+    for (const auto& instruction : instructions) {
+        auto serialized_instruction = instruction.serialize();
+        bytecode.insert(bytecode.end(),
+                        std::make_move_iterator(serialized_instruction.begin()),
+                        std::make_move_iterator(serialized_instruction.end()));
+    }
+    return bytecode;
+}
+
+std::vector<uint8_t> ControlFlow::build_bytecode(const ReturnOptions& return_options)
+{
+    current_block.finalize_with_return(
+        /*TODO(defkit) fix return size */ 1, return_options.return_value_tag, return_options.return_value_offset_index);
+    return create_bytecode(current_block.get_instructions());
+}