feat(p2p): GossibSub

cspell.json

@@ -103,6 +103,7 @@
     "fuzzers",
     "gitmodules",
     "gitrepo",
+    "gossipsub",
     "grumpkin",
     "gtest",
     "gzipped",
@@ -132,6 +133,7 @@
     "linkability",
     "lmdb",
     "maddiaa",
+    "mcache",
     "memdown",
     "memfs",
     "Merkle",
@@ -171,6 +173,7 @@
     "Palla",
     "parallelizable",
     "Pedersen",
+    "peekable",
     "permissionless",
     "permissionlessly",
     "persistable",
@@ -296,4 +299,4 @@
   "flagWords": [
     "anonymous"
   ]
-}
+}

yarn-project/aztec/src/cli/cmds/start_node.ts

@@ -71,6 +71,11 @@ export const startNode = async (
     nodeConfig = mergeEnvVarsAndCliOptions<AztecNodeConfig>(nodeConfig, parseModuleOptions(options.prover));
   }
 
+  // ensure bootstrapNodes is an array
+  if (nodeConfig.bootstrapNodes && typeof nodeConfig.bootstrapNodes === 'string') {
+    nodeConfig.bootstrapNodes = (nodeConfig.bootstrapNodes as string).split(',');
+  }
+
   if (!nodeConfig.disableSequencer && nodeConfig.disableProver) {
     throw new Error('Cannot run a sequencer without a prover');
   }

yarn-project/aztec/src/cli/texts.ts

@@ -56,6 +56,8 @@ export const cliTexts = {
     'Starts a Sequencer with options. If started additionally to --node, the Sequencer will attach to that node.\n' +
     'Available options are listed below as cliProperty:ENV_VARIABLE_NAME.\n' +
     'rcpUrl:ETHEREUM_HOST - string - The host of the Ethereum node to connect to. Default: http://localhost:8545\n' +
+    'minTxsPerBlock:SEQ_MIN_TXS_PER_BLOCK - number - The minimum number of transactions to include in a block. Default: 1\n' +
+    'maxTxsPerBlock:SEQ_MAX_TXS_PER_BLOCK - number - The maximum number of transactions to include in a block. Default: 32\n' +
     'apiKey:API_KEY - string - The key for the ethereum node if necessary.\n' +
     'chainId:CHAIN_ID - number - The chain id of the ethereum host. Default: 31337\n' +
     'version:VERSION - number - The version of the Aztec rollup. Default: 1\n' +

yarn-project/aztec/src/cli/util.ts

@@ -59,7 +59,7 @@ export const parseModuleOptions = (options: string): Record<string, string> => {
   if (!options?.length) {
     return {};
   }
-  const optionsArray = options.split(',');
+  const optionsArray = options.split(/,(?=\w+=)/);
   return optionsArray.reduce((acc, option) => {
     const [key, value] = option.split('=');
     return { ...acc, [key]: value };

yarn-project/foundation/package.json

@@ -20,6 +20,7 @@
     "./json-rpc": "./dest/json-rpc/index.js",
     "./json-rpc/server": "./dest/json-rpc/server/index.js",
     "./json-rpc/client": "./dest/json-rpc/client/index.js",
+    "./iterable": "./dest/iterable/index.js",
     "./log": "./dest/log/index.js",
     "./mutex": "./dest/mutex/index.js",
     "./fields": "./dest/fields/index.js",

yarn-project/foundation/src/iterable/all.test.ts

@@ -0,0 +1,27 @@
+import { all } from './index.js';
+
+describe('all iterable', () => {
+  it('should collect all entries of an iterator as an array', () => {
+    const values = [0, 1, 2, 3, 4];
+
+    const res = all(values);
+
+    expect(res).not.toHaveProperty('then');
+    expect(res).toEqual(values);
+  });
+
+  it('should collect all entries of an async iterator as an array', async () => {
+    const values = [0, 1, 2, 3, 4];
+
+    const generator = (async function* (): AsyncGenerator<number, void, undefined> {
+      yield* [0, 1, 2, 3, 4];
+    })();
+
+    const p = all(generator);
+    expect(p).toHaveProperty('then');
+    expect(p.then).toBeInstanceOf(Function);
+
+    const res = await p;
+    expect(res).toEqual(values);
+  });
+});

yarn-project/foundation/src/iterable/all.ts

@@ -0,0 +1,32 @@
+import { isAsyncIterable } from './isAsyncIt.js';
+
+/**
+ * Collects all values from an (async) iterable and returns them as an array
+ * @param source - Iterable to collect all values from
+ * @returns All of the iterable's values as an array.
+ */
+function all<T>(source: Iterable<T>): T[];
+function all<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T[]>;
+function all<T>(source: Iterable<T> | AsyncIterable<T>): Promise<T[]> | T[] {
+  if (isAsyncIterable(source)) {
+    return (async () => {
+      const arr = [];
+
+      for await (const entry of source) {
+        arr.push(entry);
+      }
+
+      return arr;
+    })();
+  }
+
+  const arr = [];
+
+  for (const entry of source) {
+    arr.push(entry);
+  }
+
+  return arr;
+}
+
+export { all };

yarn-project/foundation/src/iterable/filter.test.ts

@@ -0,0 +1,93 @@
+import { all, filter } from './index.js';
+
+function* values(vals: number[] = [0, 1, 2, 3, 4]): Generator<number, void, undefined> {
+  yield* vals;
+}
+
+async function* asyncValues(vals: number[] = [0, 1, 2, 3, 4]): AsyncGenerator<number, void, undefined> {
+  yield* values(vals);
+}
+
+describe('filter iterable', () => {
+  it('should filter all values greater than 2', () => {
+    const res = all(filter(values(), val => val > 2));
+
+    expect(res[Symbol.iterator]).toBeTruthy();
+    expect(res).toEqual([3, 4]);
+  });
+
+  it('should filter all values less than 2', () => {
+    const res = all(filter(values(), val => val < 2));
+
+    expect(res[Symbol.iterator]).toBeTruthy();
+    expect(res).toEqual([0, 1]);
+  });
+
+  it('should filter all values greater than 2 with a promise', () => {
+    const res = all(filter(values(), val => val > 2));
+
+    expect(res[Symbol.iterator]).toBeTruthy();
+    expect(res).toEqual([3, 4]);
+  });
+
+  it('should filter all values greater than 2 with a promise', async () => {
+    // eslint-disable-next-line require-await
+    const res = filter(values(), async val => val > 2);
+
+    expect(res[Symbol.asyncIterator]).toBeTruthy();
+    await expect(all(res)).resolves.toEqual([3, 4]);
+  });
+
+  it('should filter all async values greater than 2', async () => {
+    const res = filter(asyncValues(), val => val > 2);
+
+    expect(res[Symbol.asyncIterator]).toBeTruthy();
+    await expect(all(res)).resolves.toEqual([3, 4]);
+  });
+
+  it('should filter all async values greater than 2 with a promise', async () => {
+    // eslint-disable-next-line require-await
+    const res = filter(asyncValues(), async val => val > 2);
+
+    expect(res[Symbol.asyncIterator]).toBeTruthy();
+    await expect(all(res)).resolves.toEqual([3, 4]);
+  });
+
+  it('should filter values with indexes', () => {
+    const vals = [4, 3, 2, 1, 0];
+    const callbackArgs: any[] = [];
+    const gen = filter(values(vals), (...args: any[]) => {
+      callbackArgs.push(args);
+      return true;
+    });
+    expect(gen[Symbol.iterator]).toBeTruthy();
+
+    const results = all(gen);
+    expect(results).toHaveLength(vals.length);
+    expect(callbackArgs).toHaveLength(vals.length);
+
+    vals.forEach((value, index) => {
+      expect(callbackArgs[index][0]).toEqual(value);
+      expect(callbackArgs[index][1]).toEqual(index);
+    });
+  });
+
+  it('should filter async values with indexes', async () => {
+    const vals = [4, 3, 2, 1, 0];
+    const callbackArgs: any[] = [];
+    const gen = filter(asyncValues(vals), (...args: any[]) => {
+      callbackArgs.push(args);
+      return true;
+    });
+    expect(gen[Symbol.asyncIterator]).toBeTruthy();
+
+    const results = await all(gen);
+    expect(results).toHaveLength(vals.length);
+    expect(callbackArgs).toHaveLength(vals.length);
+
+    vals.forEach((value, index) => {
+      expect(callbackArgs[index][0]).toEqual(value);
+      expect(callbackArgs[index][1]).toEqual(index);
+    });
+  });
+});

yarn-project/foundation/src/iterable/filter.ts

@@ -0,0 +1,77 @@
+import { peek } from './peek.js';
+
+function isAsyncIterable<T>(thing: any): thing is AsyncIterable<T> {
+  return thing[Symbol.asyncIterator] != null;
+}
+
+/**
+ * Filters the passed (async) iterable by using the filter function.
+ * @param source - An iterable to filter.
+ * @returns A generator of the filtered values.
+ */
+function filter<T>(
+  source: Iterable<T>,
+  fn: (val: T, index: number) => Promise<boolean>,
+): AsyncGenerator<T, void, undefined>;
+function filter<T>(source: Iterable<T>, fn: (val: T, index: number) => boolean): Generator<T, void, undefined>;
+function filter<T>(
+  source: Iterable<T> | AsyncIterable<T>,
+  fn: (val: T, index: number) => boolean | Promise<boolean>,
+): AsyncGenerator<T, void, undefined>;
+function filter<T>(
+  source: Iterable<T> | AsyncIterable<T>,
+  fn: (val: T, index: number) => boolean | Promise<boolean>,
+): Generator<T, void, undefined> | AsyncGenerator<T, void, undefined> {
+  let index = 0;
+
+  if (isAsyncIterable(source)) {
+    return (async function* () {
+      for await (const entry of source) {
+        if (await fn(entry, index++)) {
+          yield entry;
+        }
+      }
+    })();
+  }
+
+  // if mapping function returns a promise we have to return an async generator
+  const peekable = peek(source);
+  const { value, done } = peekable.next();
+
+  if (done === true) {
+    return (function* () {})();
+  }
+
+  const res = fn(value, index++);
+
+  // @ts-expect-error .then is not present on O
+  if (typeof res.then === 'function') {
+    return (async function* () {
+      if (await res) {
+        yield value;
+      }
+
+      for await (const entry of peekable) {
+        if (await fn(entry, index++)) {
+          yield entry;
+        }
+      }
+    })();
+  }
+
+  const func = fn as (val: T, index: number) => boolean;
+
+  return (function* () {
+    if (res === true) {
+      yield value;
+    }
+
+    for (const entry of peekable) {
+      if (func(entry, index++)) {
+        yield entry;
+      }
+    }
+  })();
+}
+
+export { filter };

yarn-project/foundation/src/iterable/index.ts

@@ -0,0 +1,6 @@
+export * from './map.js';
+export * from './filter.js';
+export * from './sort.js';
+export * from './take.js';
+export * from './all.js';
+export * from './peek.js';

yarn-project/foundation/src/iterable/isAsyncIt.ts

@@ -0,0 +1,8 @@
+/**
+ * Utility function to type check an AsyncIterable
+ * @param thing - Input to type check
+ * @returns Type-checked input
+ */
+export function isAsyncIterable<T>(thing: any): thing is AsyncIterable<T> {
+  return thing[Symbol.asyncIterator] != null;
+}