From 2e2554e6a055ff7124e18d1566371d5d108c5d5d Mon Sep 17 00:00:00 2001 From: Ilyas Ridhuan Date: Mon, 11 Mar 2024 09:41:31 +0000 Subject: [PATCH] feat: initial integration avm prover (#4878) Basic shim to integrate avm simulator with the avm witgen/prover. Currently uses the bb binary and supports proving(including passing in calldata) and verifying (although both are currently WIP in the avm prover). Built on top of #4877 --- barretenberg/cpp/src/barretenberg/bb/main.cpp | 88 ++++++++++++++--- .../relations/generated/avm/avm_alu.hpp | 2 +- .../vm/avm_trace/avm_execution.cpp | 39 +++++++- .../vm/avm_trace/avm_execution.hpp | 6 +- build_manifest.yml | 2 + yarn-project/.gitignore | 1 + yarn-project/Dockerfile | 8 +- yarn-project/Dockerfile.test | 8 +- .../simulator/src/public/avm_executor.test.ts | 72 ++++++++++++++ yarn-project/simulator/src/public/executor.ts | 99 +++++++++++++++++++ 10 files changed, 304 insertions(+), 21 deletions(-) create mode 100644 yarn-project/simulator/src/public/avm_executor.test.ts diff --git a/barretenberg/cpp/src/barretenberg/bb/main.cpp b/barretenberg/cpp/src/barretenberg/bb/main.cpp index e18d365249c..a08bf91a12b 100644 --- a/barretenberg/cpp/src/barretenberg/bb/main.cpp +++ b/barretenberg/cpp/src/barretenberg/bb/main.cpp @@ -461,6 +461,70 @@ void acvm_info(const std::string& output_path) } } +/** + * @brief Writes an avm proof and corresponding (incomplete) verification key to files. + * + * Communication: + * - Filesystem: The proof and vk are written to the paths output_path/proof and output_path/vk + * + * @param bytecode_path Path to the file containing the serialised bytecode + * @param calldata_path Path to the file containing the serialised calldata (could be empty) + * @param crs_path Path to the file containing the CRS (ignition is suitable for now) + * @param output_path Path to write the output proof and verification key + */ +void avm_prove(const std::filesystem::path& bytecode_path, + const std::filesystem::path& calldata_path, + const std::string& crs_path, + const std::filesystem::path& output_path) +{ + // Get Bytecode + std::vector const avm_bytecode = read_file(bytecode_path); + std::vector call_data_bytes{}; + if (std::filesystem::exists(calldata_path)) { + call_data_bytes = read_file(calldata_path); + } + std::vector const call_data = many_from_buffer(call_data_bytes); + + srs::init_crs_factory(crs_path); + + // Prove execution and return vk + auto const [verification_key, proof] = avm_trace::Execution::prove(avm_bytecode, call_data); + // TODO(ilyas): <#4887>: Currently we only need these two parts of the vk, look into pcs_verification key reqs + std::vector vk_vector = { verification_key.circuit_size, verification_key.num_public_inputs }; + + std::filesystem::path output_vk_path = output_path.parent_path() / "vk"; + write_file(output_vk_path, to_buffer(vk_vector)); + write_file(output_path, to_buffer(proof)); +} + +/** + * @brief Verifies an avm proof and writes the result to stdout + * + * Communication: + * - stdout: The boolean value indicating whether the proof is valid. + * - proc_exit: A boolean value is returned indicating whether the proof is valid. + * + * @param proof_path Path to the file containing the serialised proof (the vk should also be in the parent) + */ +bool avm_verify(const std::filesystem::path& proof_path) +{ + std::filesystem::path vk_path = proof_path.parent_path() / "vk"; + + // Actual verification temporarily stopped (#4954) + // std::vector const proof = many_from_buffer(read_file(proof_path)); + // + // std::vector vk_bytes = read_file(vk_path); + // auto circuit_size = from_buffer(vk_bytes, 0); + // auto _num_public_inputs = from_buffer(vk_bytes, sizeof(size_t)); + // auto vk = AvmFlavor::VerificationKey(circuit_size, num_public_inputs); + // + // std::cout << avm_trace::Execution::verify(vk, proof); + // return avm_trace::Execution::verify(vk, proof); + + std::cout << 1; + return true; +} + bool flag_present(std::vector& args, const std::string& flag) { return std::find(args.begin(), args.end(), flag) != args.end(); @@ -535,22 +599,14 @@ int main(int argc, char* argv[]) std::string output_path = get_option(args, "-o", vk_path + "_fields.json"); vk_as_fields(vk_path, output_path); } else if (command == "avm_prove") { - std::string avm_bytecode_path = get_option(args, "-b", "./target/avm_bytecode.bin"); - std::string output_path = get_option(args, "-o", "./proofs/avm_proof"); - std::vector call_data_bytes{}; - - if (flag_present(args, "-d")) { - auto const call_data_path = get_option(args, "-d", "./target/call_data.bin"); - call_data_bytes = read_file(call_data_path); - } - - srs::init_crs_factory("../srs_db/ignition"); - - std::vector const call_data = many_from_buffer(call_data_bytes); - auto const avm_bytecode = read_file(avm_bytecode_path); - auto const proof = avm_trace::Execution::run_and_prove(avm_bytecode, call_data); - std::vector const proof_bytes = to_buffer(proof); - write_file(output_path, proof_bytes); + std::filesystem::path avm_bytecode_path = get_option(args, "-b", "./target/avm_bytecode.bin"); + std::filesystem::path calldata_path = get_option(args, "-d", "./target/call_data.bin"); + std::string crs_path = get_option(args, "-c", "../srs_db/ignition"); + std::filesystem::path output_path = get_option(args, "-o", "./proofs/avm_proof"); + avm_prove(avm_bytecode_path, calldata_path, crs_path, output_path); + } else if (command == "avm_verify") { + std::filesystem::path proof_path = get_option(args, "-p", "./proofs/avm_proof"); + return avm_verify(proof_path) ? 0 : 1; } else { std::cerr << "Unknown command: " << command << "\n"; return 1; diff --git a/barretenberg/cpp/src/barretenberg/relations/generated/avm/avm_alu.hpp b/barretenberg/cpp/src/barretenberg/relations/generated/avm/avm_alu.hpp index 6c98b412564..a9566f3e20d 100644 --- a/barretenberg/cpp/src/barretenberg/relations/generated/avm/avm_alu.hpp +++ b/barretenberg/cpp/src/barretenberg/relations/generated/avm/avm_alu.hpp @@ -369,4 +369,4 @@ template class avm_aluImpl { template using avm_alu = Relation>; -} // namespace bb::Avm_vm \ No newline at end of file +} // namespace bb::Avm_vm diff --git a/barretenberg/cpp/src/barretenberg/vm/avm_trace/avm_execution.cpp b/barretenberg/cpp/src/barretenberg/vm/avm_trace/avm_execution.cpp index dbd142fb2fd..a16c7a6dc15 100644 --- a/barretenberg/cpp/src/barretenberg/vm/avm_trace/avm_execution.cpp +++ b/barretenberg/cpp/src/barretenberg/vm/avm_trace/avm_execution.cpp @@ -1,5 +1,6 @@ #include "avm_execution.hpp" #include "barretenberg/common/serialize.hpp" +#include "barretenberg/flavor/generated/avm_flavor.hpp" #include "barretenberg/proof_system/circuit_builder/generated/avm_circuit_builder.hpp" #include "barretenberg/vm/avm_trace/avm_common.hpp" #include "barretenberg/vm/avm_trace/avm_deserialization.hpp" @@ -7,9 +8,11 @@ #include "barretenberg/vm/avm_trace/avm_opcode.hpp" #include "barretenberg/vm/avm_trace/avm_trace.hpp" #include "barretenberg/vm/generated/avm_composer.hpp" +#include #include #include #include +#include #include #include @@ -35,7 +38,41 @@ HonkProof Execution::run_and_prove(std::vector const& bytecode, std::ve auto composer = AvmComposer(); auto prover = composer.create_prover(circuit_builder); - return prover.construct_proof(); + auto verifier = composer.create_verifier(circuit_builder); + auto proof = prover.construct_proof(); + return proof; +} + +std::tuple Execution::prove(std::vector const& bytecode, + std::vector const& calldata) +{ + auto instructions = Deserialization::parse(bytecode); + auto trace = gen_trace(instructions, calldata); + auto circuit_builder = bb::AvmCircuitBuilder(); + circuit_builder.set_trace(std::move(trace)); + + // Temporarily use this until #4954 is resolved + assert(circuit_builder.check_circuit()); + + auto composer = AvmComposer(); + auto prover = composer.create_prover(circuit_builder); + auto verifier = composer.create_verifier(circuit_builder); + auto proof = prover.construct_proof(); + // TODO(#4887): Might need to return PCS vk when full verify is supported + return std::make_tuple(*verifier.key, proof); +} + +bool Execution::verify(AvmFlavor::VerificationKey vk, HonkProof const& proof) +{ + auto verification_key = std::make_shared(vk); + AvmVerifier verifier(verification_key); + + // todo: not needed for now until we verify the PCS/pairing of the proof + // auto pcs_verification_key = std::make_unique(verification_key->circuit_size, + // crs_factory_); + // output_state.pcs_verification_key = std::move(pcs_verification_key); + + return verifier.verify_proof(proof); } /** diff --git a/barretenberg/cpp/src/barretenberg/vm/avm_trace/avm_execution.hpp b/barretenberg/cpp/src/barretenberg/vm/avm_trace/avm_execution.hpp index 5a324eaee73..556eb4e72ae 100644 --- a/barretenberg/cpp/src/barretenberg/vm/avm_trace/avm_execution.hpp +++ b/barretenberg/cpp/src/barretenberg/vm/avm_trace/avm_execution.hpp @@ -17,6 +17,10 @@ class Execution { static std::vector gen_trace(std::vector const& instructions, std::vector const& calldata = {}); static bb::HonkProof run_and_prove(std::vector const& bytecode, std::vector const& calldata = {}); + + static std::tuple prove(std::vector const& bytecode, + std::vector const& calldata = {}); + static bool verify(AvmFlavor::VerificationKey vk, HonkProof const& proof); }; -} // namespace bb::avm_trace \ No newline at end of file +} // namespace bb::avm_trace diff --git a/build_manifest.yml b/build_manifest.yml index 0341632b1af..c07877468bc 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -155,6 +155,7 @@ yarn-project-test: - noir-packages - l1-contracts - noir-projects + - barretenberg-x86_64-linux-clang # Builds all of yarn-project, with all developer dependencies. # Creates a runnable container used to run tests and formatting checks. @@ -170,6 +171,7 @@ yarn-project: - l1-contracts - noir-projects - noir + - barretenberg-x86_64-linux-clang multiarch: host # A runnable container, sets entrypoint to be the aztec infrastructure entrypoint. diff --git a/yarn-project/.gitignore b/yarn-project/.gitignore index fccf02e939a..697a1d7b922 100644 --- a/yarn-project/.gitignore +++ b/yarn-project/.gitignore @@ -7,6 +7,7 @@ node_modules .tsbuildinfo tsconfig.tsbuildinfo .eslintcache +simulator/target .yarn/* !.yarn/patches diff --git a/yarn-project/Dockerfile b/yarn-project/Dockerfile index db5d22e899c..6026e946240 100644 --- a/yarn-project/Dockerfile +++ b/yarn-project/Dockerfile @@ -2,7 +2,8 @@ FROM --platform=linux/amd64 aztecprotocol/bb.js as bb.js FROM --platform=linux/amd64 aztecprotocol/noir-packages as noir-packages FROM --platform=linux/amd64 aztecprotocol/l1-contracts as contracts FROM --platform=linux/amd64 aztecprotocol/noir-projects as noir-projects -FROM aztecprotocol/noir as noir +FROM --platform=linux/amd64 aztecprotocol/noir as noir +FROM --platform=linux/amd64 aztecprotocol/barretenberg-x86_64-linux-clang as barretenberg FROM node:18.19.0 as builder RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean @@ -14,6 +15,11 @@ COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects # We want the native ACVM binary COPY --from=noir /usr/src/noir/noir-repo/target/release/acvm /usr/src/noir/noir-repo/target/release/acvm +COPY --from=barretenberg /usr/src/barretenberg/cpp/build/bin/bb /usr/src/barretenberg/cpp/build/bin/bb + +COPY --from=barretenberg /usr/src/barretenberg/cpp/srs_db/ /usr/src/barretenberg/cpp/srs_db/ +WORKDIR /usr/src/barretenberg/cpp/srs_db +RUN ./download_ignition.sh 0 WORKDIR /usr/src/yarn-project COPY . . diff --git a/yarn-project/Dockerfile.test b/yarn-project/Dockerfile.test index 6dfe4510760..ce29d1b6d72 100644 --- a/yarn-project/Dockerfile.test +++ b/yarn-project/Dockerfile.test @@ -2,6 +2,7 @@ FROM --platform=linux/amd64 aztecprotocol/bb.js as bb.js FROM --platform=linux/amd64 aztecprotocol/noir-packages as noir-packages FROM --platform=linux/amd64 aztecprotocol/l1-contracts as contracts FROM --platform=linux/amd64 aztecprotocol/noir-projects as noir-projects +FROM --platform=linux/amd64 aztecprotocol/barretenberg-x86_64-linux-clang as barretenberg FROM node:18.19.0 as builder RUN apt update && apt install -y jq curl perl && rm -rf /var/lib/apt/lists/* && apt-get clean @@ -11,6 +12,11 @@ COPY --from=bb.js /usr/src/barretenberg/ts /usr/src/barretenberg/ts COPY --from=noir-packages /usr/src/noir/packages /usr/src/noir/packages COPY --from=contracts /usr/src/l1-contracts /usr/src/l1-contracts COPY --from=noir-projects /usr/src/noir-projects /usr/src/noir-projects +COPY --from=barretenberg /usr/src/barretenberg/cpp/build/bin/bb /usr/src/barretenberg/cpp/build/bin/bb + +COPY --from=barretenberg /usr/src/barretenberg/cpp/srs_db/ /usr/src/barretenberg/cpp/srs_db/ +WORKDIR /usr/src/barretenberg/cpp/srs_db +RUN ./download_ignition.sh 0 WORKDIR /usr/src/yarn-project COPY . . @@ -31,4 +37,4 @@ RUN yarn prepare:check && yarn formatting && yarn test # Avoid pushing some huge container back to ecr. FROM scratch -COPY --from=builder /usr/src/yarn-project/README.md /usr/src/yarn-project/README.md \ No newline at end of file +COPY --from=builder /usr/src/yarn-project/README.md /usr/src/yarn-project/README.md diff --git a/yarn-project/simulator/src/public/avm_executor.test.ts b/yarn-project/simulator/src/public/avm_executor.test.ts new file mode 100644 index 00000000000..aaff6232443 --- /dev/null +++ b/yarn-project/simulator/src/public/avm_executor.test.ts @@ -0,0 +1,72 @@ +import { AztecAddress, CallContext, EthAddress, FunctionData, FunctionSelector, Header } from '@aztec/circuits.js'; +import { makeHeader } from '@aztec/circuits.js/testing'; +import { Fr } from '@aztec/foundation/fields'; +import { AvmTestContractArtifact } from '@aztec/noir-contracts.js'; + +import { MockProxy, mock } from 'jest-mock-extended'; + +import { CommitmentsDB, PublicContractsDB, PublicStateDB } from './db.js'; +import { PublicExecution } from './execution.js'; +import { PublicExecutor } from './executor.js'; + +describe('AVM WitGen and Proof Generation', () => { + let publicState: MockProxy; + let publicContracts: MockProxy; + let commitmentsDb: MockProxy; + let header: Header; + + const callContext = CallContext.from({ + msgSender: AztecAddress.random(), + storageContractAddress: AztecAddress.random(), + portalContractAddress: EthAddress.random(), + functionSelector: FunctionSelector.empty(), + isContractDeployment: false, + isDelegateCall: false, + isStaticCall: false, + startSideEffectCounter: 0, + }); + const contractAddress = AztecAddress.random(); + + beforeEach(() => { + publicState = mock(); + publicContracts = mock(); + commitmentsDb = mock(); + + const randomInt = Math.floor(Math.random() * 1000000); + header = makeHeader(randomInt); + }, 10000); + + it('Should prove valid execution of bytecode that performs addition', async () => { + const args: Fr[] = [new Fr(1), new Fr(2)]; + // Bytecode for the following contract is encoded: + // const bytecode = encodeToBytecode([ + // new CalldataCopy(/*indirect=*/ 0, /*cdOffset=*/ 0, /*copySize=*/ 2, /*dstOffset=*/ 0), + // new Add(/*indirect=*/ 0, TypeTag.FIELD, /*aOffset=*/ 0, /*bOffset=*/ 1, /*dstOffset=*/ 2), + // new Return(/*indirect=*/ 0, /*returnOffset=*/ 2, /*copySize=*/ 1), + // ]); + const bytecode: Buffer = Buffer.from('HwAAAAAAAAAAAgAAAAAAAAYAAAAAAAAAAQAAAAI3AAAAAAIAAAAB', 'base64'); + publicContracts.getBytecode.mockResolvedValue(bytecode); + const executor = new PublicExecutor(publicState, publicContracts, commitmentsDb, header); + const functionData = FunctionData.empty(); + const execution: PublicExecution = { contractAddress, functionData, args, callContext }; + const [proof, vk] = await executor.getAvmProof(execution); + const valid = await executor.verifyAvmProof(vk, proof); + expect(valid).toBe(true); + }); + + // This is skipped as we require MOV to be implemented in the AVM + it.skip('Should prove valid execution contract function that performs addition', async () => { + const args: Fr[] = [new Fr(1), new Fr(2)]; + + const addArtifact = AvmTestContractArtifact.functions.find(f => f.name === 'avm_addArgsReturn')!; + const bytecode = Buffer.from(addArtifact.bytecode, 'base64'); + publicContracts.getBytecode.mockResolvedValue(bytecode); + const functionData = FunctionData.fromAbi(addArtifact); + const execution: PublicExecution = { contractAddress, functionData, args, callContext }; + + const executor = new PublicExecutor(publicState, publicContracts, commitmentsDb, header); + const [proof, vk] = await executor.getAvmProof(execution); + const valid = await executor.verifyAvmProof(vk, proof); + expect(valid).toBe(true); + }); +}); diff --git a/yarn-project/simulator/src/public/executor.ts b/yarn-project/simulator/src/public/executor.ts index 42cd5ec6ac0..5e307e3a45f 100644 --- a/yarn-project/simulator/src/public/executor.ts +++ b/yarn-project/simulator/src/public/executor.ts @@ -2,6 +2,10 @@ import { FunctionL2Logs } from '@aztec/circuit-types'; import { GlobalVariables, Header, PublicCircuitPublicInputs } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; +import { spawn } from 'child_process'; +import fs from 'fs/promises'; +import path from 'path'; + import { Oracle, acvm, extractCallStack, extractReturnWitness } from '../acvm/index.js'; import { AvmContext } from '../avm/avm_context.js'; import { AvmMachineState } from '../avm/avm_machine_state.js'; @@ -220,4 +224,99 @@ export class PublicExecutor { const newWorldState = context.persistableState.flush(); return temporaryConvertAvmResults(execution, newWorldState, result); } + + /** + * These functions are currently housed in the temporary executor as it relies on access to + * oracles like the contractsDB and this is the least intrusive way to achieve this. + * When we remove this executor(tracking issue #4792) and have an interface that is compatible with the kernel circuits, + * this will be moved to sequencer-client/prover. + */ + + /** + * Generates a proof for an associated avm execution. This is currently only used for testing purposes, + * as proof generation is not fully complete in the AVM yet. + * @param execution - The execution to run. + * @returns An AVM proof and the verification key. + */ + public async getAvmProof(avmExecution: PublicExecution): Promise { + // The paths for the barretenberg binary and the write path are hardcoded for now. + // We additionally need the path to a valid crs for proof generation. + // const bbPath = '../../barretenberg/cpp'; + const bbPath = path.resolve('../../barretenberg/cpp'); + const artifactsPath = path.resolve('target'); + + // Create the directory if it does not exist + await fs.mkdir(artifactsPath, { recursive: true }); + + const calldataPath = path.join(artifactsPath, 'calldata.bin'); + const bytecodePath = path.join(artifactsPath, 'avm_bytecode.bin'); + const proofPath = path.join(artifactsPath, 'proof'); + + const { args, functionData, contractAddress } = avmExecution; + const bytecode = await this.contractsDb.getBytecode(contractAddress, functionData.selector); + // Write call data and bytecode to files. + await Promise.all([ + fs.writeFile( + calldataPath, + args.map(c => c.toBuffer()), + ), + fs.writeFile(bytecodePath, bytecode!), + ]); + + const bbBinary = spawn(path.join(bbPath, 'build', 'bin', 'bb'), [ + 'avm_prove', + '-b', + bytecodePath, + '-d', + calldataPath, + '-c', + path.join(bbPath, 'srs_db', 'ignition'), + '-o', + proofPath, + ]); + // The binary writes the proof and the verification key to the write path. + return new Promise((resolve, reject) => { + bbBinary.on('close', () => { + resolve(Promise.all([fs.readFile(proofPath), fs.readFile(path.join(artifactsPath, 'vk'))])); + }); + // Catch and propagate errors from spawning + bbBinary.on('error', err => { + reject(err); + }); + }); + } + /** + * Verifies an AVM proof. This function is currently only used for testing purposes, as verification + * is not fully complete in the AVM yet. + * @param vk - The verification key to use. + * @param proof - The proof to verify. + * @returns True if the proof is valid, false otherwise. + */ + async verifyAvmProof(vk: Buffer, proof: Buffer): Promise { + // The relative paths for the barretenberg binary and the write path are hardcoded for now. + const bbPath = path.resolve('../../barretenberg/cpp'); + const artifactsPath = path.resolve('./target'); + + const vkPath = path.join(artifactsPath, 'vk'); + const proofPath = path.join(artifactsPath, 'proof'); + + // Write the verification key and the proof to files. + await Promise.all([fs.writeFile(vkPath, vk), fs.writeFile(proofPath, proof)]); + + const bbBinary = spawn(path.join(bbPath, 'build', 'bin', 'bb'), ['avm_verify', '-p', proofPath]); + // The binary prints to stdout 1 if the proof is valid and 0 if it is not. + return new Promise((resolve, reject) => { + let result = Buffer.alloc(0); + bbBinary.stdout.on('data', data => { + result += data; + }); + bbBinary.on('close', () => { + resolve(result.toString() === '1'); + }); + // Catch and propagate errors from spawning + bbBinary.on('error', err => { + reject(err); + }); + }); + } }