Improve fetching

Fetch source from GitHub

Make source fetching optional

Fetch executable solc from GitHub

docs/api/server/verification1/verify.md

@@ -78,7 +78,7 @@ If using `application/json`, the files should be in an object under the key `fil
 
 **Condition** : Failed fetching missing files. OR Contract bytecode does not match deployed bytecode.
 
-**Code** : `500 Internal Server Error`
+**Code** : `400 Bad Request Error`
 
 **Content** : 
 ```json

docs/api/server/verification2/exchange-object.md

@@ -44,7 +44,8 @@ The object expected by the server from the client is a proper subset of (1) and
             "networkId": "100",
             "compilerVersion": "0.6.6+commit.6c089d02"
         }
-    ]
+    ],
+    "fetch": true // if true or "true", then considered true, all other values are treated as false
 }
 ```
 - If the client does not know some of the properties yet, they may be omitted.

docs/api/server/verification2/verify-validated.md

@@ -13,9 +13,9 @@
 
 ## Responses
 
-**Assumptions for all example responses (except the last one)** :
-* There is one pending contract with all source files, but no `address` or `networkId`.
-* Supplying the following minimum object (extra properties would be ignored):
+**Assumptions for all example responses (except the last)** :
+* There is one pending contract with 0 or 1 missing source file, but no `address` or `networkId`.
+* Supplying the following minimum object:
 ```json
 {
     "contracts": [
@@ -29,7 +29,9 @@
 }
 ```
 
-**Condition** : The provided contract `perfect`ly matches the one at the provided `networkId` and `address`.
+**Conditions** :
+- All the source files have been previously provided.
+- The provided contract `perfect`ly matches the one at the provided `networkId` and `address`.
 
 **Code** : `200 OK`
 
@@ -60,6 +62,41 @@
 
 ### OR
 
+**Conditions** :
+- One source file is missing.
+- Fetching requested by providing `fetch: true`, look [here](exchange-object.md) for more info.
+- The provided contract `perfect`ly matches the one at the provided `networkId` and `address`.
+
+**Code** : `200 OK`
+
+**Content** :
+
+```json
+{
+    "contracts": [
+        {
+            "verificationId": "0x3f67e9f57515bb1e7195c7c5af1eff630091567c0bb65ba3dece57a56da766fe",
+            "compiledPath": "browser/1_Storage.sol",
+            "name": "Storage",
+            "compilerVersion": "0.6.6+commit.6c089d02",
+            "address": "0x656d0062eC89c940213E3F3170EA8b2add1c0143",
+            "networkId": "100",
+            "files": {
+                "found": [
+                    "browser/1_Storage.sol"
+                ],
+                "missing": []
+            },
+            "status": "perfect"
+        }
+    ],
+    "unused": [],
+    "fetch": true
+}
+```
+
+### OR
+
 **Condition** : The contract at the provided `networkId` and `address` has already been verified at `2021-01-12T15:41:56.502Z`.
 
 **Code** : `200 OK`

services/core/src/utils/CheckedContract.ts

@@ -43,10 +43,15 @@ export class CheckedContract {
     name: string;
 
     /** Checks whether this contract is valid or not.
+     *  This is a static method due to persistence issues.
+     * 
+     * @param contract the contract to be checked
+     * @param ignoreMissing a flag indicating that missing sources should be ignored
      * @returns true if no sources are missing or are invalid (malformed); false otherwise
      */
-    public static isValid(contract: CheckedContract): boolean {
-        return isEmpty(contract.missing) && isEmpty(contract.invalid);
+    public static isValid(contract: CheckedContract, ignoreMissing?: boolean): boolean {
+        return (isEmpty(contract.missing) || ignoreMissing)
+            && isEmpty(contract.invalid);
     }
 
     private sourceMapToStringMap(input: SourceMap) {
@@ -196,65 +201,51 @@ export class CheckedContract {
      * 
      * @param log log object
      */
-    public async fetchMissing(log?: bunyan): Promise<void> {
+    public static async fetchMissing(contract: CheckedContract, log?: bunyan): Promise<void> {
         const retrieved: StringMap = {};
-        for (const fileName in this.missing) {
-            if (!fileName.startsWith("@openzeppelin")) {
-                continue;
-            }
-
-            const file = this.missing[fileName];
-            const hash = this.missing[fileName].keccak256;
-
-            let success = false;
-            for (const url of file.urls) {
-                if (url.startsWith(IPFS_PREFIX)) {
-                    const ipfsCode = url.slice(IPFS_PREFIX.length);
-                    const ipfsUrl = 'https://ipfs.infura.io:5001/api/v0/cat?arg='+ipfsCode;
-                    const retrievedContent = await this.performFetch(ipfsUrl, hash, fileName, log);
-                    if (retrievedContent) {
-                        success = true;
-                        retrieved[fileName] = retrievedContent;
-                        break;
+        const missingFiles: string[] = [];
+        for (const fileName in contract.missing) {
+            const file = contract.missing[fileName];
+            const hash = contract.missing[fileName].keccak256;
+
+            let retrievedContent = null;
+
+            const githubUrl = getGithubUrl(fileName);
+            if (githubUrl) {
+                retrievedContent = await performFetch(githubUrl, hash, fileName, log);
+
+            } else {
+                for (const url of file.urls) {
+                    if (url.startsWith(IPFS_PREFIX)) {
+                        const ipfsCode = url.slice(IPFS_PREFIX.length);
+                        const ipfsUrl = 'https://ipfs.infura.io:5001/api/v0/cat?arg='+ipfsCode;
+                        retrievedContent = await performFetch(ipfsUrl, hash, fileName, log);
+                        if (retrievedContent) {
+                            break;
+                        }
                     }
                 }
             }
+
+            if (retrievedContent) {
+                retrieved[fileName] = retrievedContent;
+            } else {
+                missingFiles.push(fileName);
+                break; // makes an early exit
+            }
         }
 
         for (const fileName in retrieved) {
-            delete this.missing[fileName];
-            this.solidity[fileName] = retrieved[fileName];
+            delete contract.missing[fileName];
+            contract.solidity[fileName] = retrieved[fileName];
         }
 
-        const missingFiles = Object.keys(this.missing);
         if (missingFiles.length) {
             log.error({ loc: "[FETCH]", contractName: this.name, missingFiles });
-            throw new Error(`Missing sources after fetching: ${missingFiles.join(", ")}`);
-        }
-    }
-
-    private async performFetch(url: string, hash: string, fileName: string, log?: bunyan): Promise<string> {
-        const infoObject = { loc: "[FETCH]", fileName, url };
-        if (log) log.info(infoObject, "Fetch attempt");
-
-        const res = await fetch(url);
-        if (res.status === 200) {
-            const content = await res.text();
-            if (Web3.utils.keccak256(content) !== hash) {
-                if (log) log.error(infoObject, "The calculated and the provided hash don't match.");
-                return null;
-            }
-
-            if (log) log.info(infoObject, "Fetch successful!");
-            return content;
-
-        } else {
-            if (log) log.error(infoObject, "Fetch failed!");
-            return null;
+            throw new Error(`Resource missing; unsuccessful fetching: ${missingFiles.join(", ")}`);
         }
     }
 
-
     /**
      * Returns a message describing the errors encountered while validating the metadata.
      * Does not include a trailing newline.
@@ -264,4 +255,49 @@ export class CheckedContract {
     public getInfo() {
         return CheckedContract.isValid(this) ? this.composeSuccessMessage() : this.composeErrorMessage();
     }
+}
+
+/**
+ * Performs fetch and compares with the hash provided.
+ * 
+ * @param url the url to be used as the file source
+ * @param hash the hash of the file to be fetched; used for later comparison
+ * @param fileName the name of the file; used for logging
+ * @param log whether or not to log
+ * @returns the fetched file if found; null otherwise
+ */
+async function performFetch(url: string, hash: string, fileName: string, log?: bunyan): Promise<string> {
+    const infoObject = { loc: "[FETCH]", fileName, url };
+    if (log) log.info(infoObject, "Fetch attempt");
+
+    const res = await fetch(url);
+    if (res.status === 200) {
+        const content = await res.text();
+        if (Web3.utils.keccak256(content) !== hash) {
+            if (log) log.error(infoObject, "The calculated and the provided hash don't match.");
+            return null;
+        }
+
+        if (log) log.info(infoObject, "Fetch successful!");
+        return content;
+
+    } else {
+        if (log) log.error(infoObject, "Fetch failed!");
+        return null;
+    }
+}
+
+/**
+ * Makes a GitHub-compatible url out of the provided url, if possible.
+ * 
+ * @param url 
+ * @returns a GitHub-compatible url if possible; null otherwise
+ */
+function getGithubUrl(url: string): string {
+    if (!url.includes("github.com")) {
+        return null;
+    }
+    return url
+        .replace("github.com", "raw.githubusercontent.com")
+        .replace("/blob/", "/");
 }

services/core/src/utils/types.ts

@@ -12,6 +12,7 @@ export interface InputData {
     addresses: string[],
     contract?: CheckedContract,
     bytecode?: string;
+    fetchMissing?: boolean;
 }
 
 export interface CompilationSettings {

services/verification/src/services/Injector.ts

@@ -195,7 +195,7 @@ export class Injector {
      * @return {Promise<object>}              address & status of successfully verified contract
      */
     public async inject(inputData: InputData): Promise<Match> {
-        const { chain, addresses, contract } = inputData;
+        const { chain, addresses, contract, fetchMissing } = inputData;
         this.validateAddresses(addresses);
         this.validateChain(chain);
 
@@ -209,8 +209,8 @@ export class Injector {
 
         if (!CheckedContract.isValid(contract)) {
             // eslint-disable-next-line no-useless-catch
-            try {
-                await contract.fetchMissing(this.log);
+            if (fetchMissing) try {
+                await CheckedContract.fetchMissing(contract, this.log);
             } catch(err) {
                 throw err;
             }

services/verification/src/utils.ts

@@ -6,6 +6,10 @@ import fs from 'fs';
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const solc = require('solc');
 import {spawnSync} from 'child_process';
+import { StatusCodes } from 'http-status-codes';
+import { mkdirpSync } from 'fs-extra';
+
+const GITHUB_SOLC_REPO = "https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/";
 
 export interface RecompilationResult {
     bytecode: string,
@@ -102,7 +106,8 @@ export async function recompile(
 }
 
 /**
- * Searches for a solc: first for a local executable version, then using the getSolc function.
+ * Searches for a solc: first for a local executable version, then from GitHub
+ * and then using the getSolc function.
  * Once the compiler is retrieved, it is used, and the stringified solc output is returned.
  * 
  * @param version the version of solc to be used for compilation
@@ -113,9 +118,14 @@ export async function recompile(
 async function useCompiler(version: string, input: any, log: {info: any, error: any}) {
     const inputStringified = JSON.stringify(input);
     const repoPath = process.env.SOLC_REPO || "solc-repo";
-    const solcPath = Path.join(repoPath, `solc-linux-amd64-v${version}`);
+    const fileName = `solc-linux-amd64-v${version}`;
+    const solcPath = Path.join(repoPath, fileName);
     let compiled: string = null;
 
+    if (!fs.existsSync(solcPath)) {
+        await fetchSolcFromGitHub(solcPath, version, fileName, log);
+    }
+
     if (fs.existsSync(solcPath)) {
         log.info({loc: "[RECOMPILE]", version, solcPath}, "Compiling with external executable");
         const shellOutputBuffer = spawnSync(solcPath, ["--standard-json"], {input: inputStringified});
@@ -128,6 +138,23 @@ async function useCompiler(version: string, input: any, log: {info: any, error:
     return compiled;
 }
 
+async function fetchSolcFromGitHub(solcPath: string, version: string, fileName: string, log: {info: any, error: any}) {
+    const githubSolcURI = GITHUB_SOLC_REPO + encodeURIComponent(fileName);
+    const logObject = {loc: "[RECOMPILE]", version, githubSolcURI};
+    log.info(logObject, "Fetching executable solc from GitHub");
+
+    const res = await fetch(githubSolcURI);
+    if (res.status === StatusCodes.OK) {
+        log.info(logObject, "Successfully fetched executable solc from GitHub");
+        mkdirpSync(Path.dirname(solcPath));
+        const buffer = await res.buffer();
+        fs.writeFileSync(solcPath, buffer, { mode: 0o755 });
+
+    } else {
+        log.error(logObject, "Failed fetching executable solc from GitHub");
+    }
+}
+
 /**
  * Removes post-fixed metadata from a bytecode string
  * (for partial bytecode match comparisons )

src/server/controllers/VerificationController-util.ts

@@ -39,7 +39,8 @@ export type SessionMaps = {
 export type MySession = 
     Session &
     SessionMaps & { 
-    unusedSources: string[]
+    unusedSources: string[],
+    fetch?: boolean
 };
 
 export type MyRequest = 
@@ -59,9 +60,9 @@ export type SendableContract =
     verificationId?: string
 }
 
-export function isVerifiable(contractWrapper: ContractWrapper) {
+export function isVerifiable(contractWrapper: ContractWrapper, ignoreMissing?: boolean) {
     const contract = contractWrapper.contract;
-    return isEmpty(contract.missing)
+    return (isEmpty(contract.missing) || ignoreMissing)
         && isEmpty(contract.invalid)
         && Boolean(contractWrapper.compilerVersion)
         && Boolean(contractWrapper.address)
@@ -103,7 +104,7 @@ export function getSessionJSON(session: MySession) {
     }
 
     const unused = session.unusedSources || [];
-    return { contracts, unused };
+    return { contracts, unused, fetch: session.fetch };
 }
 
 export function generateId(obj: any): string {