feat: improved DX (#258)

* update some packages

* utilizing `node_modules/.cache`

* implement metaMaskDownloader progress messages

* improve CI

* implement downloader CLI

.github/workflows/main.yml

@@ -9,6 +9,9 @@ on:
     branches:
       - '**'
 
+env:
+  METAMASK_CACHE_PATH: 'node_modules/.cache/.metamask'
+
 jobs:
   build:
     name: Build
@@ -23,11 +26,23 @@ jobs:
         run: yarn --prefer-offline --frozen-lockfile
       - name: Build
         run: yarn build
+  download:
+    needs: build
+    name: 'Download Metamask extension'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          cache: 'yarn'
+      - name: Install deps
+        run: yarn --prefer-offline --frozen-lockfile
       - name: Download metamask
         run: yarn run download:metamask
       - uses: actions/cache/save@v3
         with:
-          path: .metamask
+          path: ${{env.METAMASK_CACHE_PATH}}
           key: metamask-extension
   lint:
     needs: build
@@ -44,7 +59,7 @@ jobs:
       - name: Lint
         run: yarn run lint
   tests:
-    needs: build
+    needs: [build, download]
     name: Tests
     runs-on: ubuntu-latest
     strategy:
@@ -60,7 +75,7 @@ jobs:
           cache: 'yarn'
       - uses: actions/cache/restore@v3
         with:
-          path: .metamask
+          path: ${{env.METAMASK_CACHE_PATH}}
           key: metamask-extension
       - name: Install
         run: yarn --prefer-offline --frozen-lockfile
@@ -74,7 +89,7 @@ jobs:
   tests-user-data:
     name: Tests UserData
     runs-on: ubuntu-latest
-    needs: tests
+    needs: [tests, download]
     strategy:
       fail-fast: false
       matrix:
@@ -85,6 +100,10 @@ jobs:
         with:
           node-version: "16"
           cache: 'yarn'
+      - uses: actions/cache/restore@v3
+        with:
+          path: ${{env.METAMASK_CACHE_PATH}}
+          key: metamask-extension
       - name: Install
         run: yarn --prefer-offline --frozen-lockfile
       - name: Tests UserData

bin/metamask_downloader.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+"use strict"
+
+require("../dist/bin/downloader.js");

package.json

@@ -27,6 +27,9 @@
     "test:playwright:flask": "AUTOMATION=playwright yarn run test:flask",
     "download:metamask": "node -r ts-node/register ./scripts/download_metamask.ts"
   },
+  "bin": {
+    "mmd": "./bin/metamask_downloader.js"
+  },
   "repository": {
     "type": "git",
     "url": "https://github.com/chainsafe/dappeteer.git"
@@ -68,12 +71,12 @@
     "eslint": "^8.24.0",
     "ganache": "^7.4.3",
     "jest-environment-node": "^27.1.1",
-    "mocha": "^10.0.0",
+    "mocha": "^10.2.0",
     "playwright": "^1.29.2",
     "prettier": "^2.2.1",
     "puppeteer": "14.0.0",
     "solc": "0.5.2",
-    "ts-node": "10.8.1",
+    "ts-node": "^10.9.1",
     "typescript": "~4.7",
     "web3": "4.0.1-alpha.2",
     "web3-eth-contract": "4.0.1-alpha.2"

src/bin/downloader.ts

@@ -0,0 +1,30 @@
+import path from "path";
+import { RECOMMENDED_METAMASK_VERSION } from "../constants";
+import downloader, {
+  defaultDirectory,
+} from "../setup/utils/metaMaskDownloader";
+
+console.log(`Running MetaMask extension downloader`, "\n");
+
+let flask = false;
+let version = RECOMMENDED_METAMASK_VERSION;
+let destination = defaultDirectory;
+
+for (let i = 2; i < process.argv.length; i++) {
+  const arg = process.argv[i];
+  if (arg === "--flask") flask = true;
+  else if (arg === "--version" || arg === "-v") version = process.argv[++i];
+  else if (arg === "--destination" || arg === "-d")
+    destination = path.resolve(process.argv[++i]);
+  else {
+    console.error(`Unknown argument ${arg}`);
+    process.exit();
+  }
+}
+
+void (async function () {
+  await downloader(version, {
+    flask,
+    location: destination,
+  });
+})();

src/setup/utils/metaMaskDownloader.ts

@@ -7,10 +7,8 @@ import * as path from "path";
 import StreamZip from "node-stream-zip";
 
 export const defaultDirectory = path.resolve(
-  __dirname,
-  "..",
-  "..",
-  "..",
+  "node_modules",
+  ".cache",
   ".metamask"
 );
 
@@ -35,21 +33,34 @@ export default async (
       ? location
       : location?.download || path.resolve(defaultDirectory, "download");
 
+  console.log(
+    `Getting MetaMask ${
+      options.flask ? "flask" : ""
+    } extension version: ${version}`
+  );
+  console.log(`extension stored in directory: ${metaMaskDirectory}`);
+  console.log(`downloaded files stored in: ${downloadDirectory}`, "\n");
+
   if (version !== "latest") {
     let filename = version.replace(/\./g, "_");
     if (options?.flask) {
       filename = "flask_" + filename;
     }
     const extractDestination = path.resolve(metaMaskDirectory, filename);
-    if (fs.existsSync(extractDestination)) return extractDestination;
+    if (fs.existsSync(extractDestination)) {
+      console.log(
+        "Found already available extension files - skipping download"
+      );
+      return extractDestination;
+    }
   }
   const { filename, downloadUrl, tag } = await getMetaMaskReleases(
     version,
     options?.flask ?? false
   );
   let destFilename = tag.replace(/\./g, "_");
   if (options?.flask) {
-    destFilename = "flask_" + filename;
+    destFilename = "flask_" + destFilename;
   }
   const extractDestination = path.resolve(metaMaskDirectory, destFilename);
   if (!fs.existsSync(extractDestination)) {
@@ -58,21 +69,23 @@ export default async (
       downloadUrl,
       downloadDirectory
     );
+    console.log("Unpacking release");
     const zip = new StreamZip.async({ file: downloadedFile });
     fs.mkdirSync(extractDestination);
     await zip.extract(null, extractDestination);
+    console.log("Unpack successful");
+  } else {
+    console.log("Found already available extension files - skipping download");
   }
   return extractDestination;
 };
 
-// eslint-disable-next-line @typescript-eslint/naming-convention
 const request = (url: string): Promise<IncomingMessage> =>
   new Promise((resolve) => {
     const request = get(url, (response) => {
       if (response.statusCode == 302) {
         const redirectRequest = get(response.headers.location, resolve);
         redirectRequest.on("error", (error) => {
-          // eslint-disable-next-line no-console
           console.warn("request redirected error:", error.message);
           throw error;
         });
@@ -81,7 +94,6 @@ const request = (url: string): Promise<IncomingMessage> =>
       }
     });
     request.on("error", (error) => {
-      // eslint-disable-next-line no-console
       console.warn("request error:", error.message);
       throw error;
     });
@@ -97,11 +109,13 @@ const downloadMetaMaskReleases = (
     if (!fs.existsSync(location)) {
       fs.mkdirSync(location, { recursive: true });
     }
+    console.log("Downloading MetaMask release");
     const fileLocation = path.join(location, name);
     const file = fs.createWriteStream(fileLocation);
     const stream = await request(url);
     stream.pipe(file);
     stream.on("end", () => {
+      console.log("Download successful");
       resolve(fileLocation);
     });
   });
@@ -114,7 +128,7 @@ const getMetaMaskReleases = (
   flask: boolean
 ): Promise<MetaMaskReleases> =>
   new Promise((resolve, reject) => {
-    // eslint-disable-next-line @typescript-eslint/naming-convention
+    console.log("Searching for MetaMask release");
     const request = get(
       metaMaskReleasesUrl,
       { headers: { "User-Agent": "Mozilla/5.0" } },
@@ -131,17 +145,16 @@ const getMetaMaskReleases = (
             if (
               version === "latest" ||
               result.name.includes(version) ||
-              // eslint-disable-next-line @typescript-eslint/no-unsafe-call
               result.tag_name.includes(version)
             ) {
               for (const asset of result.assets) {
-                // eslint-disable-next-line @typescript-eslint/no-unsafe-call
                 if (
                   (!flask && asset.name.includes("chrome")) ||
                   (flask &&
                     asset.name.includes("flask") &&
                     asset.name.includes("chrome"))
                 ) {
+                  console.log("Found requested MetaMask release");
                   resolve({
                     downloadUrl: asset.browser_download_url,
                     filename: asset.name,
@@ -156,7 +169,6 @@ const getMetaMaskReleases = (
       }
     );
     request.on("error", (error) => {
-      // eslint-disable-next-line no-console
       console.warn("getMetaMaskReleases error:", error.message);
       throw error;
     });

yarn.lock

@@ -1773,11 +1773,6 @@
     "@typescript-eslint/types" "5.39.0"
     eslint-visitor-keys "^3.3.0"
 
-"@ungap/[email protected]":
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz#aa58042711d6e3275dd37dc597e5d31e8c290a44"
-  integrity sha512-sL/cEvJWAnClXw0wHk85/2L0G6Sj8UB0Ctc1TEMbKSsmpRosqhwj9gWgFRZSrBr2f9tiXISwNhCPmlfqUqyb9Q==
-
 JSONStream@^1.0.3:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0"
@@ -4397,12 +4392,11 @@ mkdirp@^1.0.4:
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
   integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
 
-mocha@^10.0.0:
-  version "10.0.0"
-  resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.0.0.tgz#205447d8993ec755335c4b13deba3d3a13c4def9"
-  integrity sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA==
+mocha@^10.2.0:
+  version "10.2.0"
+  resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.2.0.tgz#1fd4a7c32ba5ac372e03a17eef435bd00e5c68b8"
+  integrity sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==
   dependencies:
-    "@ungap/promise-all-settled" "1.1.2"
     ansi-colors "4.1.1"
     browser-stdout "1.3.1"
     chokidar "3.5.3"
@@ -5750,10 +5744,10 @@ tr46@~0.0.3:
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
   integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
 
-ts-node@10.8.1:
-  version "10.8.1"
-  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.8.1.tgz#ea2bd3459011b52699d7e88daa55a45a1af4f066"
-  integrity sha512-Wwsnao4DQoJsN034wePSg5nZiw4YKXf56mPIAeD6wVmiv+RytNSWqc2f3fKvcUoV+Yn2+yocD71VOfQHbmVX4g==
+ts-node@^10.9.1:
+  version "10.9.1"
+  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b"
+  integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==
   dependencies:
     "@cspotcode/source-map-support" "^0.8.0"
     "@tsconfig/node10" "^1.0.7"