chore: Convert repo to a monorepo

.gitignore

@@ -3,10 +3,13 @@ node_modules
 ArbTokenLists
 FullList
 dist
+.idea
+.DS_Store
 
 # Logs
 logs
 *.log
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+

package.json

@@ -1,46 +1,11 @@
 {
   "name": "arb-token-lists",
+  "private": true,
   "version": "1.0.0",
   "main": "index.js",
+  "author": "Offchain Labs, Inc.",
   "license": "MIT",
-  "scripts": {
-    "build": "tsc",
-    "main": "ts-node src/main.ts",
-    "lint": "eslint .",
-    "format": "prettier './**/*.{js,json,md,ts,yml}' --write && yarn run lint --fix",
-    "update": "yarn ts-node src/main.ts --action update",
-    "arbify": "yarn ts-node src/main.ts --action arbify",
-    "permit": "yarn ts-node src/main.ts --action permit",
-    "fullList": "yarn ts-node src/main.ts --action full --tokenList full",
-    "updateNova": "yarn ts-node src/main.ts --action update --l2NetworkID 42170",
-    "novaify": "yarn ts-node src/main.ts --action arbify --l2NetworkID 42170",
-    "fullNova": "yarn ts-node src/main.ts --action full --tokenList full --l2NetworkID 42170",
-    "start_server": "yarn ts-node server/app.ts"
-  },
-  "dependencies": {
-    "@arbitrum/sdk": "^3.0.0",
-    "@uniswap/token-lists": "^1.0.0-beta.30",
-    "ajv": "^8.6.3",
-    "ajv-formats": "^2.1.1",
-    "axios": "^0.23.0",
-    "better-ajv-errors": "^1.1.2",
-    "dotenv": "^16.0.3",
-    "eslint-config-prettier": "^8.3.0",
-    "eslint-plugin-prettier": "^4.0.0",
-    "express": "^4.17.1",
-    "graphql": "^15.6.1",
-    "graphql-request": "^3.6.1",
-    "prettier": "^2.4.1",
-    "ts-node": "^10.8.1",
-    "typescript": "^4.7.3",
-    "yargs": "^17.2.1"
-  },
-  "devDependencies": {
-    "@types/express": "^4.17.13",
-    "@types/node": "^16.11.1",
-    "@types/yargs": "^17.0.4",
-    "@typescript-eslint/eslint-plugin": "^5.1.0",
-    "@typescript-eslint/parser": "^5.1.0",
-    "eslint": "^8.0.1"
-  }
+  "workspaces": [
+    "packages/*"
+  ]
 }

.env.sample → packages/cli/.env.sample

@@ -1,4 +1,6 @@
 MAINNET_RPC="https://mainnet.infura.io/v3/<infura key>"
 ARB_ONE_RPC="https://arb1.arbitrum.io/rpc"
+GOERLI_RPC="https://goerli-rollup.arbitrum.io/rpc"
+RINKEBY_RPC="https://rinkeby.arbitrum.io/rpc"
 l2NetworkID=42161
-PORT=3000
+PORT=3000

packages/cli/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "cli",
+  "version": "1.0.0",
+  "main": "main.js",
+  "author": "Offchain Labs, Inc.",
+  "license": "MIT",
+  "scripts": {
+    "arbify": "yarn ts-node src/main.ts --action arbify",
+    "arbify:nova": "yarn ts-node src/main.ts --action arbify --l2NetworkID 42170",
+    "build": "rm -rf ./dist && tsc",
+    "format": "prettier --config-precedence file-override --write \"src/**/*.{tsx,ts,md}\" && yarn run lint",
+    "fullList": "yarn ts-node src/main.ts --action full --tokenList full",
+    "fullList:nova": "yarn ts-node src/main.ts --action full --tokenList full --l2NetworkID 42170",
+    "lint": "tsc --noEmit && eslint src/**/*.ts --fix",
+    "lint:fix": "tsc --noEmit && eslint 'src/**/*.{js,ts,tsx}' --quiet --fix",
+    "main": "ts-node src/main.ts",
+    "permit": "yarn ts-node src/main.ts --action permit",
+    "update": "yarn ts-node src/main.ts --action update",
+    "update:all": "./scripts/update_all",
+    "update:nova": "yarn ts-node src/main.ts --action update --l2NetworkID 42170",
+    "update:src_lists": "./scripts/update_src_lists",
+    "watch": "tsc --watch"
+  },
+  "dependencies": {
+    "yargs": "^17.2.1",
+    "dotenv": "^16.0.3",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-prettier": "^4.0.0",
+    "prettier": "^2.4.1",
+    "sdk": "1.0.0"
+  },
+  "devDependencies": {
+    "@types/yargs": "^17.0.4",
+    "@typescript-eslint/eslint-plugin": "^5.1.0",
+    "@typescript-eslint/parser": "^5.1.0",
+    "eslint": "^8.0.1"
+  }
+}

update_src_lists → packages/cli/scripts/update_src_lists

@@ -1,2 +1,4 @@
+#!/bin/bash
+
 yarn arbify --tokenList ./src/SourceLists/whitelist_era_src.json --l2NetworkID 42161 --includeOldDataFields true
 yarn arbify --tokenList ./src/SourceLists/rinkeby_src.json --l2NetworkID 421611  

src/lib/getClargs.ts → packages/cli/src/getClargs.ts

@@ -14,11 +14,10 @@
  * limitations under the License.
  */
 
-// import yargs from 'yargs/yargs';
-'use strict';
 import yargs from 'yargs/yargs';
+import { hideBin } from 'yargs/helpers';
 
-const argv = yargs(process.argv.slice(2))
+const argv = yargs(hideBin(process.argv))
   .options({
     l2NetworkID: {
       type: 'number',
@@ -30,7 +29,7 @@ const argv = yargs(process.argv.slice(2))
       type: 'boolean',
     },
     action: {
-      type: 'string',
+      choices: ['arbify', 'full', 'permit', 'update'],
     },
     includePermitTags: {
       type: 'boolean',

packages/cli/src/main.ts

@@ -0,0 +1,58 @@
+import {
+  ArbTokenList,
+  addPermitTags,
+  arbifyL1List,
+  generateFullList,
+  updateArbifiedList,
+  writeToFile,
+} from 'sdk';
+
+import args from './getClargs';
+
+const main = async () => {
+  const {
+    action,
+    includeOldDataFields,
+    includePermitTags,
+    l2NetworkID,
+    tokenList: fullTokenList,
+  } = args;
+
+  if (action === 'full') {
+    if (fullTokenList !== 'full')
+      throw new Error("expected --tokenList 'full'");
+    if (includePermitTags)
+      throw new Error('full list mode does not support permit tagging');
+
+    return writeToFile(await generateFullList(l2NetworkID), l2NetworkID);
+  }
+
+  let tokenList: ArbTokenList;
+
+  switch (action) {
+    case 'arbify':
+      tokenList = await arbifyL1List(
+        fullTokenList,
+        l2NetworkID,
+        !!includeOldDataFields
+      );
+      break;
+    case 'update':
+      tokenList = await updateArbifiedList(fullTokenList, l2NetworkID);
+      break;
+    default:
+      throw new Error(`action ${action} not recognised`);
+  }
+
+  if (includePermitTags)
+    tokenList = await addPermitTags(tokenList, l2NetworkID);
+
+  writeToFile(tokenList, l2NetworkID);
+};
+
+main()
+  .then(() => console.log('Done.'))
+  .catch(err => {
+    console.error(err);
+    throw err;
+  });

packages/cli/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": ["src"],
+  "compilerOptions": {
+    "noEmit": true
+  }
+}

packages/sdk/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "sdk",
+  "version": "1.0.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "author": "Offchain Labs, Inc.",
+  "license": "MIT",
+  "scripts": {
+    "build": "rm -rf ./dist && tsc",
+    "watch": "tsc --watch",
+    "lint": "tsc --noEmit && eslint src/**/*.ts --fix",
+    "lint:fix": "tsc --noEmit && eslint 'src/**/*.{js,ts,tsx}' --quiet --fix",
+    "format": "prettier --config-precedence file-override --write \"src/**/*.{tsx,ts,md}\" && yarn run lint",
+    "start_server": "yarn ts-node server/app.ts"
+  },
+  "files": [
+    "dist/**/*"
+  ],
+  "dependencies": {
+    "@arbitrum/sdk": "^3.0.0",
+    "@uniswap/token-lists": "^1.0.0-beta.30",
+    "ajv": "^8.6.3",
+    "ajv-formats": "^2.1.1",
+    "axios": "^0.23.0",
+    "better-ajv-errors": "^1.1.2",
+    "dotenv": "^16.0.3",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-prettier": "^4.0.0",
+    "graphql": "^15.6.1",
+    "graphql-request": "^3.6.1",
+    "prettier": "^2.4.1",
+    "ts-node": "^10.8.1",
+    "typescript": "^4.7.3",
+    "express": "^4.17.1"
+  },
+  "devDependencies": {
+    "@types/node": "^16.11.1",
+    "@typescript-eslint/eslint-plugin": "^5.1.0",
+    "@typescript-eslint/parser": "^5.1.0",
+    "eslint": "^8.0.1",
+    "@types/express": "^4.17.13"
+  }
+}

server/app.ts → packages/sdk/server/app.ts

@@ -1,15 +1,16 @@
 import express from 'express';
-const app = express();
 import dotenv from 'dotenv';
 import { readFileSync } from 'fs';
+
+const app = express();
 dotenv.config();
 const port = process.env.PORT || 8080;
 
-app.get('/', (req, res) => {
+app.get('/', (_, res) => {
   res.send('ping');
 });
 
-app.get('/all_tokens', (req, res) => {
+app.get('/all_tokens', (_, res) => {
   const tokenData = readFileSync('./src/FullList/all_tokens.json');
 
   res.send(JSON.parse(tokenData.toString()));

packages/sdk/src/Assets/index.ts

@@ -0,0 +1,4 @@
+import coingecko_uris from './coingecko_uris.json';
+import logo_uris from './logo_uris.json';
+
+export { coingecko_uris, logo_uris };

src/PermitTokens/permitSignature.ts → packages/sdk/src/PermitTokens/permitSignature.ts

@@ -151,10 +151,11 @@ enum PermitTypes {
 }
 
 export const addPermitTags = async (
-  tokenList: ArbTokenList
+  tokenList: ArbTokenList,
+  l2NetworkID: number
 ): Promise<ArbTokenList> => {
   console.log('Adding permit tags');
-  const { l1, l2 } = await getNetworkConfig();
+  const { l1, l2 } = await getNetworkConfig(l2NetworkID);
 
   const value = utils.parseUnits('1.0', 18);
   const deadline = constants.MaxUint256;

packages/sdk/src/index.ts

@@ -0,0 +1,6 @@
+export * from './lib';
+export * from './PermitTokens/permitSignature';
+export * from './Assets';
+
+import all_tokens from './FullList/all_tokens.json';
+export { all_tokens };

src/lib/graph.ts → packages/sdk/src/lib/graph.ts

@@ -1,5 +1,4 @@
 import { request, gql } from 'graphql-request';
-import { isNova } from './utils';
 import { GraphTokenResult, GraphTokensResult } from './types';
 import { excludeList } from './constants';
 
@@ -48,65 +47,6 @@ const graphGatewayBlockNumField = (networkID: string | number) => {
   return +networkID === 421613 ? 'l2BlockNum' : 'blockNum';
 };
 
-export const getTokens = async (
-  tokenList: { addr: string; logo: string | undefined }[],
-  _networkID: string | number
-): Promise<Array<GraphTokenResult>> => {
-  if (isNova) {
-    console.warn('empty subgraph for nova');
-    return [];
-  }
-  const networkID =
-    typeof _networkID === 'number' ? _networkID.toString() : _networkID;
-  const clientUrl = chaidIdToGraphClientUrl(networkID);
-  // lazy solution for big lists for now; we'll have to paginate once we have > 500 tokens registed
-  if (tokenList.length > 500) {
-    const allTokens = await getAllTokens(networkID);
-    const allTokenAddresses = new Set(
-      allTokens.map(token => token.l1TokenAddr.toLowerCase())
-    );
-    tokenList = tokenList.filter(token =>
-      allTokenAddresses.has(token.addr.toLowerCase())
-    );
-    if (tokenList.length > 500)
-      throw new Error('Too many tokens for graph query');
-  }
-  const formattedAddresses = tokenList
-    .map(token => `"${token.addr}"`.toLowerCase())
-    .join(',');
-  const blockNumber = graphGatewayBlockNumField(_networkID);
-  const query = gql`
-  {
-    tokens(first: 500, skip: 0, where:{
-      id_in:[${formattedAddresses}]
-    }) {
-      l1TokenAddr: id
-      joinTableEntry: gateway(
-        first: 1
-        orderBy: ${blockNumber}
-        orderDirection: desc
-      ) {
-        id
-        ${blockNumber}
-        token {
-          tokenAddr: id
-        }
-        gateway {
-          gatewayAddr: id
-        }
-      }
-    }
-  }
-`;
-
-  const { tokens } = (await request(clientUrl, query)) as GraphTokensResult;
-  tokens.map(token => isGraphTokenResult(token));
-
-  return tokens.filter(
-    token => !excludeList.includes(token.l1TokenAddr.toLowerCase())
-  );
-};
-
 export const getAllTokens = async (
   _networkID: string | number
 ): Promise<Array<GraphTokenResult>> => {

packages/sdk/src/lib/index.ts

@@ -0,0 +1,7 @@
+export * from './constants';
+export * from './graph';
+export * from './instantiate_bridge';
+export * from './store';
+export * from './token_list_gen';
+export * from './types';
+export * from './utils';

src/lib/instantiate_bridge.ts → packages/sdk/src/lib/instantiate_bridge.ts

@@ -1,14 +1,14 @@
 import { providers } from 'ethers';
-import args from './getClargs';
 import dotenv from 'dotenv';
 import { getL2Network, MultiCaller } from '@arbitrum/sdk';
+
 dotenv.config();
 
-const networkID = args.l2NetworkID || 42161;
+export const getNetworkConfig = async (l2NetworkID: number) => {
+  const networkID = l2NetworkID ?? 42161;
 
-console.log('Using L2 networkID:', networkID);
+  console.log('Using L2 networkID:', networkID);
 
-export const getNetworkConfig = async () => {
   const l2Rpc = (() => {
     if (networkID === 42161) return 'https://arb1.arbitrum.io/rpc';
     else if (networkID === 421611) return 'https://rinkeby.arbitrum.io/rpc';