chore: Add optional artifical delay for test prover (#7832)

To test long-running proving jobs without requiring a massive amount of
RAM. Adds a new `PROVER_TEST_DELAY_MS` env var that defaults to zero,
only used in the `TestCircuitProver`.

yarn-project/aztec/src/cli/cmds/start_prover_agent.ts

@@ -28,18 +28,9 @@ export const startProverAgent: ServiceStarter = async (options, signalHandlers,
     if (!proverConfig.acvmBinaryPath || !proverConfig.bbBinaryPath) {
       throw new Error('Cannot start prover without simulation or native prover options');
     }
-
-    circuitProver = await BBNativeRollupProver.new(
-      {
-        acvmBinaryPath: proverConfig.acvmBinaryPath,
-        bbBinaryPath: proverConfig.bbBinaryPath,
-        acvmWorkingDirectory: proverConfig.acvmWorkingDirectory,
-        bbWorkingDirectory: proverConfig.bbWorkingDirectory,
-      },
-      telemetry,
-    );
+    circuitProver = await BBNativeRollupProver.new(proverConfig, telemetry);
   } else {
-    circuitProver = new TestCircuitProver(telemetry);
+    circuitProver = new TestCircuitProver(telemetry, undefined, proverConfig);
   }
 
   const agent = new ProverAgent(

yarn-project/bb-prover/src/test/test_circuit_prover.ts

@@ -33,6 +33,7 @@ import {
   makeRecursiveProof,
 } from '@aztec/circuits.js';
 import { createDebugLogger } from '@aztec/foundation/log';
+import { sleep } from '@aztec/foundation/sleep';
 import { Timer } from '@aztec/foundation/timer';
 import {
   ProtocolCircuitVkIndexes,
@@ -70,11 +71,12 @@ import { mapPublicKernelToCircuitName } from '../stats.js';
 export class TestCircuitProver implements ServerCircuitProver {
   private wasmSimulator = new WASMSimulator();
   private instrumentation: ProverInstrumentation;
+  private logger = createDebugLogger('aztec:test-prover');
 
   constructor(
     telemetry: TelemetryClient,
     private simulationProvider?: SimulationProvider,
-    private logger = createDebugLogger('aztec:test-prover'),
+    private opts: { proverTestDelayMs: number } = { proverTestDelayMs: 0 },
   ) {
     this.instrumentation = new ProverInstrumentation(telemetry, 'TestCircuitProver');
   }
@@ -103,7 +105,7 @@ export class TestCircuitProver implements ServerCircuitProver {
       SimulatedServerCircuitArtifacts.PrivateKernelEmptyArtifact,
     );
     const result = convertSimulatedPrivateKernelEmptyOutputsFromWitnessMap(witness);
-
+    await this.delay();
     return makePublicInputsAndRecursiveProof(
       result,
       makeRecursiveProof(NESTED_RECURSIVE_PROOF_LENGTH),
@@ -131,6 +133,7 @@ export class TestCircuitProver implements ServerCircuitProver {
       SimulatedServerCircuitArtifacts.PrivateKernelEmptyArtifact,
     );
     const result = convertPrivateKernelEmptyOutputsFromWitnessMap(witness);
+    await this.delay();
     return makePublicInputsAndRecursiveProof(
       result,
       makeRecursiveProof(NESTED_RECURSIVE_PROOF_LENGTH),
@@ -171,7 +174,7 @@ export class TestCircuitProver implements ServerCircuitProver {
       result.toBuffer().length,
       this.logger,
     );
-
+    await this.delay();
     return Promise.resolve(rootParityInput);
   }
 
@@ -210,7 +213,7 @@ export class TestCircuitProver implements ServerCircuitProver {
       result.toBuffer().length,
       this.logger,
     );
-
+    await this.delay();
     return Promise.resolve(rootParityInput);
   }
 
@@ -242,20 +245,22 @@ export class TestCircuitProver implements ServerCircuitProver {
       result.toBuffer().length,
       this.logger,
     );
+    await this.delay();
     return makePublicInputsAndRecursiveProof(
       result,
       makeRecursiveProof(NESTED_RECURSIVE_PROOF_LENGTH),
       ProtocolCircuitVks['BaseRollupArtifact'],
     );
   }
 
-  public getTubeProof(
+  public async getTubeProof(
     _tubeInput: TubeInputs,
   ): Promise<{ tubeVK: VerificationKeyData; tubeProof: RecursiveProof<typeof TUBE_PROOF_LENGTH> }> {
-    return Promise.resolve({
+    await this.delay();
+    return {
       tubeVK: VerificationKeyData.makeFake(),
       tubeProof: makeEmptyRecursiveProof(TUBE_PROOF_LENGTH),
-    });
+    };
   }
 
   /**
@@ -286,6 +291,7 @@ export class TestCircuitProver implements ServerCircuitProver {
       result.toBuffer().length,
       this.logger,
     );
+    await this.delay();
     return makePublicInputsAndRecursiveProof(
       result,
       makeEmptyRecursiveProof(NESTED_RECURSIVE_PROOF_LENGTH),
@@ -321,6 +327,7 @@ export class TestCircuitProver implements ServerCircuitProver {
       result.toBuffer().length,
       this.logger,
     );
+    await this.delay();
     return makePublicInputsAndRecursiveProof(
       result,
       makeEmptyRecursiveProof(NESTED_RECURSIVE_PROOF_LENGTH),
@@ -354,7 +361,7 @@ export class TestCircuitProver implements ServerCircuitProver {
       result.toBuffer().length,
       this.logger,
     );
-
+    await this.delay();
     return makePublicInputsAndRecursiveProof(
       result,
       makeEmptyRecursiveProof(NESTED_RECURSIVE_PROOF_LENGTH),
@@ -383,19 +390,26 @@ export class TestCircuitProver implements ServerCircuitProver {
       result.toBuffer().length,
       this.logger,
     );
-
+    await this.delay();
     return makePublicInputsAndRecursiveProof(
       result,
       makeEmptyRecursiveProof(NESTED_RECURSIVE_PROOF_LENGTH),
       ProtocolCircuitVks['PublicKernelTailArtifact'],
     );
   }
 
-  getAvmProof(_inputs: AvmCircuitInputs): Promise<ProofAndVerificationKey> {
+  public async getAvmProof(_inputs: AvmCircuitInputs): Promise<ProofAndVerificationKey> {
     // We can't simulate the AVM because we don't have enough context to do so (e.g., DBs).
     // We just return an empty proof and VK data.
     this.logger.debug('Skipping AVM simulation in TestCircuitProver.');
-    return Promise.resolve({ proof: makeEmptyProof(), verificationKey: VerificationKeyData.makeFake() });
+    await this.delay();
+    return { proof: makeEmptyProof(), verificationKey: VerificationKeyData.makeFake() };
+  }
+
+  private async delay(): Promise<void> {
+    if (this.opts.proverTestDelayMs > 0) {
+      await sleep(this.opts.proverTestDelayMs);
+    }
   }
 
   // Not implemented for test circuits

yarn-project/circuit-types/src/interfaces/prover-client.ts

@@ -24,6 +24,8 @@ export type ProverConfig = {
   proverJobTimeoutMs: number;
   /** The interval to check job health status */
   proverJobPollIntervalMs: number;
+  /** Artificial delay to introduce to all operations to the test prover. */
+  proverTestDelayMs: number;
   /** Identifier of the prover */
   proverId?: Fr;
 };
@@ -70,6 +72,11 @@ export const proverConfigMappings: ConfigMappingsType<ProverConfig> = {
     parseEnv: (val: string) => parseProverId(val),
     description: 'Identifier of the prover',
   },
+  proverTestDelayMs: {
+    env: 'PROVER_TEST_DELAY_MS',
+    description: 'Artificial delay to introduce to all operations to the test prover.',
+    ...numberConfigHelper(0),
+  },
 };
 
 function parseProverId(str: string) {

yarn-project/foundation/src/config/env_var.ts

@@ -76,6 +76,7 @@ export type EnvVar =
   | 'PROVER_PUBLISH_RETRY_INTERVAL_MS'
   | 'PROVER_PUBLISHER_PRIVATE_KEY'
   | 'PROVER_REQUIRED_CONFIRMATIONS'
+  | 'PROVER_TEST_DELAY_MS'
   | 'TX_PROVIDER_NODE_URL'
   | 'TXE_PORT'
   | 'BOT_PXE_URL'