chore: big synching case + stability (#9022)

- Moves `cheat_code` log from `info` to `verbose`
- Hardens fixture creation, 
- Create fixture for 1000 blocks of 4 private transfers
- Some plumbing to make it work

Hardens the fixture creation by introducing a "timejump" before the
first L2 block will land. By making the timejump a fixed jump ahead of a
starting time, we can ensure that we have some fixed upper limit and as
long as we are performing the setup within it we should be good.
In this case, that period is 200 blocks.

Beware, that the `Rollup` contract MUST be deployed at the same time,
since the `GENESIS_TIME` is used for all future slot computations, e.g.,
messing with that is pain. So if possible, please only change add stuff
to the L1 deployments AFTER the rollup deployment.

Since the big fixture of 1000 blocks is big and our system is very slow,
it will be skipped unless the `RUN_THE_BIG_ONE` flag is set.

```
aztec:e2e_synching [INFO] Stats: blockCount: 1000, txCount: 4, complexity: PrivateTransfer: {"numberOfBlocks":1008,"syncTime":225.7148497581482} +4m
```

yarn-project/aztec.js/src/utils/cheat_codes.ts

@@ -125,7 +125,7 @@ export class EthCheatCodes {
     if (res.error) {
       throw new Error(`Error mining: ${res.error.message}`);
     }
-    this.logger.info(`Mined ${numberOfBlocks} blocks`);
+    this.logger.verbose(`Mined ${numberOfBlocks} blocks`);
   }
 
   /**
@@ -138,7 +138,7 @@ export class EthCheatCodes {
     if (res.error) {
       throw new Error(`Error setting balance for ${account}: ${res.error.message}`);
     }
-    this.logger.info(`Set balance for ${account} to ${balance}`);
+    this.logger.verbose(`Set balance for ${account} to ${balance}`);
   }
 
   /**
@@ -150,7 +150,7 @@ export class EthCheatCodes {
     if (res.error) {
       throw new Error(`Error setting block interval: ${res.error.message}`);
     }
-    this.logger.info(`Set block interval to ${interval}`);
+    this.logger.verbose(`Set block interval to ${interval}`);
   }
 
   /**
@@ -162,7 +162,7 @@ export class EthCheatCodes {
     if (res.error) {
       throw new Error(`Error setting next block timestamp: ${res.error.message}`);
     }
-    this.logger.info(`Set next block timestamp to ${timestamp}`);
+    this.logger.verbose(`Set next block timestamp to ${timestamp}`);
   }
 
   /**
@@ -175,7 +175,7 @@ export class EthCheatCodes {
       throw new Error(`Error warping: ${res.error.message}`);
     }
     await this.mine();
-    this.logger.info(`Warped to ${timestamp}`);
+    this.logger.verbose(`Warped to ${timestamp}`);
   }
 
   /**
@@ -189,7 +189,7 @@ export class EthCheatCodes {
     }
     const jsonContent = JSON.stringify(res.result);
     fs.writeFileSync(`${fileName}.json`, jsonContent, 'utf8');
-    this.logger.info(`Dumped state to ${fileName}`);
+    this.logger.verbose(`Dumped state to ${fileName}`);
   }
 
   /**
@@ -202,7 +202,7 @@ export class EthCheatCodes {
     if (res.error) {
       throw new Error(`Error loading state: ${res.error.message}`);
     }
-    this.logger.info(`Loaded state from ${fileName}`);
+    this.logger.verbose(`Loaded state from ${fileName}`);
   }
 
   /**
@@ -228,7 +228,7 @@ export class EthCheatCodes {
     if (res.error) {
       throw new Error(`Error setting storage for contract ${contract} at ${slot}: ${res.error.message}`);
     }
-    this.logger.info(`Set storage for contract ${contract} at ${slot} to ${value}`);
+    this.logger.verbose(`Set storage for contract ${contract} at ${slot} to ${value}`);
   }
 
   /**
@@ -252,7 +252,7 @@ export class EthCheatCodes {
     if (res.error) {
       throw new Error(`Error impersonating ${who}: ${res.error.message}`);
     }
-    this.logger.info(`Impersonating ${who}`);
+    this.logger.verbose(`Impersonating ${who}`);
   }
 
   /**
@@ -264,7 +264,7 @@ export class EthCheatCodes {
     if (res.error) {
       throw new Error(`Error when stopping the impersonation of ${who}: ${res.error.message}`);
     }
-    this.logger.info(`Stopped impersonating ${who}`);
+    this.logger.verbose(`Stopped impersonating ${who}`);
   }
 
   /**
@@ -277,7 +277,7 @@ export class EthCheatCodes {
     if (res.error) {
       throw new Error(`Error setting bytecode for ${contract}: ${res.error.message}`);
     }
-    this.logger.info(`Set bytecode for ${contract} to ${bytecode}`);
+    this.logger.verbose(`Set bytecode for ${contract} to ${bytecode}`);
   }
 
   /**

yarn-project/end-to-end/src/e2e_synching.test.ts

@@ -67,6 +67,7 @@ import { type EndToEndContext, getPrivateKeyFromIndex, setup, setupPXEService }
 const SALT = 420;
 const AZTEC_GENERATE_TEST_DATA = !!process.env.AZTEC_GENERATE_TEST_DATA;
 const START_TIME = 1893456000; // 2030 01 01 00 00
+const RUN_THE_BIG_ONE = !!process.env.RUN_THE_BIG_ONE;
 
 const MINT_AMOUNT = 1000n;
 
@@ -338,6 +339,7 @@ const variants: VariantDefinition[] = [
   { blockCount: 10, txCount: 36, txComplexity: TxComplexity.PrivateTransfer },
   { blockCount: 10, txCount: 36, txComplexity: TxComplexity.PublicTransfer },
   { blockCount: 10, txCount: 9, txComplexity: TxComplexity.Spam },
+  { blockCount: 1000, txCount: 4, txComplexity: TxComplexity.PrivateTransfer },
 ];
 
 describe('e2e_synching', () => {
@@ -349,10 +351,23 @@ describe('e2e_synching', () => {
       if (!AZTEC_GENERATE_TEST_DATA) {
         return;
       }
+
+      // @note  If the `RUN_THE_BIG_ONE` flag is not set, we DO NOT run it.
+      if (!RUN_THE_BIG_ONE && variantDef.blockCount === 1000) {
+        return;
+      }
+
       const variant = new TestVariant(variantDef);
 
       // The setup is in here and not at the `before` since we are doing different setups depending on what mode we are running in.
-      const { teardown, pxe, sequencer, aztecNode, wallet } = await setup(1, { salt: SALT, l1StartTime: START_TIME });
+      // We require that at least 200 eth blocks have passed from the START_TIME before we see the first L2 block
+      // This is to keep the setup more stable, so as long as the setup is less than 100 L1 txs, changing the setup should not break the setup
+      const { teardown, pxe, sequencer, aztecNode, wallet } = await setup(1, {
+        salt: SALT,
+        l1StartTime: START_TIME,
+        l2StartTime: START_TIME + 200 * ETHEREUM_SLOT_DURATION,
+        assumeProvenThrough: 10 + variant.blockCount,
+      });
       variant.setPXE(pxe as PXEService);
 
       // Deploy a token, such that we could use it
@@ -381,12 +396,13 @@ describe('e2e_synching', () => {
       await variant.writeBlocks(blocks);
       await teardown();
     },
-    2_400_000,
+    240_400_000,
   );
 
   const testTheVariant = async (
     variant: TestVariant,
     alternativeSync: (opts: Partial<EndToEndContext>, variant: TestVariant) => Promise<void>,
+    assumeProvenThrough: number = Number.MAX_SAFE_INTEGER,
   ) => {
     if (AZTEC_GENERATE_TEST_DATA) {
       return;
@@ -397,6 +413,7 @@ describe('e2e_synching', () => {
         salt: SALT,
         l1StartTime: START_TIME,
         skipProtocolContracts: true,
+        assumeProvenThrough,
       });
 
     await (aztecNode as any).stop();
@@ -427,6 +444,7 @@ describe('e2e_synching', () => {
       while ((await cheatCodes.eth.timestamp()) < targetTime) {
         await cheatCodes.eth.mine();
       }
+      // If it breaks here, first place you should look is the pruning.
       await publisher.proposeL2Block(block);
     }
 
@@ -436,32 +454,45 @@ describe('e2e_synching', () => {
   };
 
   describe('replay history and then do a fresh sync', () => {
-    it.each(variants)('vanilla - %s', async (variantDef: VariantDefinition) => {
-      await testTheVariant(
-        new TestVariant(variantDef),
-        async (opts: Partial<EndToEndContext>, variant: TestVariant) => {
-          // All the blocks have been "re-played" and we are now to simply get a new node up to speed
-          const timer = new Timer();
-          const freshNode = await AztecNodeService.createAndSync(
-            { ...opts.config!, disableSequencer: true, disableValidator: true },
-            new NoopTelemetryClient(),
-          );
-          const syncTime = timer.s();
+    it.each(variants)(
+      'vanilla - %s',
+      async (variantDef: VariantDefinition) => {
+        // @note  If the `RUN_THE_BIG_ONE` flag is not set, we DO NOT run it.
+        if (!RUN_THE_BIG_ONE && variantDef.blockCount === 1000) {
+          return;
+        }
 
-          const blockNumber = await freshNode.getBlockNumber();
+        await testTheVariant(
+          new TestVariant(variantDef),
+          async (opts: Partial<EndToEndContext>, variant: TestVariant) => {
+            // All the blocks have been "re-played" and we are now to simply get a new node up to speed
+            const timer = new Timer();
+            const freshNode = await AztecNodeService.createAndSync(
+              { ...opts.config!, disableSequencer: true, disableValidator: true },
+              new NoopTelemetryClient(),
+            );
+            const syncTime = timer.s();
 
-          opts.logger!.info(
-            `Stats: ${variant.description()}: ${JSON.stringify({
-              numberOfBlocks: blockNumber,
-              syncTime,
-            })}`,
-          );
-        },
-      );
-    });
+            const blockNumber = await freshNode.getBlockNumber();
+
+            opts.logger!.info(
+              `Stats: ${variant.description()}: ${JSON.stringify({
+                numberOfBlocks: blockNumber,
+                syncTime,
+              })}`,
+            );
+
+            await freshNode.stop();
+          },
+        );
+      },
+      RUN_THE_BIG_ONE ? 600_000 : 300_000,
+    );
   });
 
   describe('a wild prune appears', () => {
+    const ASSUME_PROVEN_THROUGH = 0;
+
     it('archiver following catches reorg as it occur and deletes blocks', async () => {
       if (AZTEC_GENERATE_TEST_DATA) {
         return;
@@ -557,7 +588,10 @@ describe('e2e_synching', () => {
           [LogType.NOTEENCRYPTED, LogType.ENCRYPTED, LogType.UNENCRYPTED].forEach(async t => {
             expect(await archiver.getLogs(blockTip.number, 1, t)).toEqual([]);
           });
+
+          await archiver.stop();
         },
+        ASSUME_PROVEN_THROUGH,
       );
     });
 
@@ -621,6 +655,7 @@ describe('e2e_synching', () => {
           await aztecNode.stop();
           await watcher.stop();
         },
+        ASSUME_PROVEN_THROUGH,
       );
     });
 
@@ -679,6 +714,7 @@ describe('e2e_synching', () => {
           await aztecNode.stop();
           await watcher.stop();
         },
+        ASSUME_PROVEN_THROUGH,
       );
     });
   });

yarn-project/end-to-end/src/fixtures/utils.ts

@@ -314,6 +314,10 @@ export type SetupOptions = {
   l1BlockTime?: number;
   /** Anvil Start time */
   l1StartTime?: number;
+  /** The anvil time where we should at the earliest be seeing L2 blocks */
+  l2StartTime?: number;
+  /** How far we should assume proven */
+  assumeProvenThrough?: number;
 } & Partial<AztecNodeConfig>;
 
 /** Context for an end-to-end test as returned by the `setup` function */
@@ -420,12 +424,18 @@ export async function setup(
       config.l1RpcUrl,
       publisherHdAccount!,
       logger,
-      { salt: opts.salt, initialValidators: opts.initialValidators },
+      { salt: opts.salt, initialValidators: opts.initialValidators, assumeProvenThrough: opts.assumeProvenThrough },
       chain,
     ));
 
   config.l1Contracts = deployL1ContractsValues.l1ContractAddresses;
 
+  if (opts.l2StartTime) {
+    // This should only be used in synching test or when you need to have a stable
+    // timestamp for the first l2 block.
+    await ethCheatCodes.warp(opts.l2StartTime);
+  }
+
   const watcher = new AnvilTestWatcher(
     new EthCheatCodes(config.l1RpcUrl),
     deployL1ContractsValues.l1ContractAddresses.rollupAddress,