From b53d17082a9c2ba059599b9df0b743ecdba7d9c9 Mon Sep 17 00:00:00 2001 From: FabijanC Date: Tue, 19 Jan 2021 11:22:34 +0100 Subject: [PATCH] Improve fetching Fetch source from GitHub Make source fetching optional Fetch executable solc from GitHub --- Sourcify.postman_collection.json | 401 +++++++++++++++--- docs/api/server/verification1/verify.md | 2 +- .../server/verification2/exchange-object.md | 3 +- .../server/verification2/verify-validated.md | 45 +- services/core/src/utils/CheckedContract.ts | 130 ++++-- services/core/src/utils/types.ts | 1 + .../verification/src/services/Injector.ts | 6 +- services/verification/src/utils.ts | 31 +- .../VerificationController-util.ts | 9 +- .../controllers/VerificationController.ts | 44 +- test/server.js | 146 +++++-- 11 files changed, 657 insertions(+), 161 deletions(-) diff --git a/Sourcify.postman_collection.json b/Sourcify.postman_collection.json index e81d9643d..34436fc42 100644 --- a/Sourcify.postman_collection.json +++ b/Sourcify.postman_collection.json @@ -74,86 +74,79 @@ } }, "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } }, { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ] - }, - { - "name": "Verification API 2", - "item": [ - { - "name": "Add input files - success", + "name": "verify - missing - fetch", "request": { "method": "POST", "header": [], "body": { "mode": "formdata", "formdata": [ + { + "key": "address", + "value": "0x656d0062eC89c940213E3F3170EA8b2add1c0143", + "type": "text" + }, + { + "key": "chain", + "value": "100", + "type": "text" + }, { "key": "files", "type": "file", - "src": [ - "test/testcontracts/1_Storage/1_Storage.sol", - "test/testcontracts/1_Storage/metadata.json" - ] + "src": "test/testcontracts/1_Storage/metadata.json" + }, + { + "key": "fetch", + "value": "true", + "type": "text" } ] }, "url": { - "raw": "{{host}}/input-files", + "raw": "{{host}}", "host": [ "{{host}}" - ], - "path": [ - "input-files" ] } }, "response": [ { - "name": "Add input files - success", + "name": "verify - missing - fetch", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "formdata", "formdata": [ + { + "key": "address", + "value": "0x656d0062eC89c940213E3F3170EA8b2add1c0143", + "type": "text" + }, + { + "key": "chain", + "value": "100", + "type": "text" + }, { "key": "files", "type": "file", - "src": [ - "test/testcontracts/1_Storage/1_Storage.sol", - "test/testcontracts/1_Storage/metadata.json" - ] + "src": "test/testcontracts/1_Storage/metadata.json" + }, + { + "key": "fetch", + "value": "true", + "type": "text" } ] }, "url": { - "raw": "{{host}}/input-files", + "raw": "{{host}}", "host": [ "{{host}}" - ], - "path": [ - "input-files" ] } }, @@ -175,19 +168,19 @@ }, { "key": "Content-Length", - "value": "287" + "value": "88" }, { "key": "ETag", - "value": "W/\"11f-g6j9TfRz4+JLOVlzsIMF7fjowkQ\"" + "value": "W/\"58-J/J2ShSm6vDzcetM7QhLTil+co8\"" }, { "key": "Set-Cookie", - "value": "sourcify_vid=s%3Ai8fGEBmscCNl4g5Lr-55tEiTPRIFJZI3.i%2Bg%2B9doxHuunTqq5V6nPPsnkc5GIc8RUpuzVm2wFGqo; Path=/; Expires=Wed, 13 Jan 2021 05:40:19 GMT; HttpOnly" + "value": "sourcify_vid=s%3A1MUnH1uhcDyJytHrXoYrnU3pnYpHwg4k.ESJevK%2BKUpvwiC9MxqT%2FJTXaU%2BGKSIv3yYqJlJ4W6MA; Path=/; Expires=Wed, 20 Jan 2021 23:50:51 GMT; HttpOnly" }, { "key": "Date", - "value": "Tue, 12 Jan 2021 17:40:19 GMT" + "value": "Wed, 20 Jan 2021 11:50:51 GMT" }, { "key": "Connection", @@ -195,22 +188,157 @@ } ], "cookie": [], - "body": "{\n \"contracts\": [\n {\n \"verificationId\": \"0x3f67e9f57515bb1e7195c7c5af1eff630091567c0bb65ba3dece57a56da766fe\",\n \"compiledPath\": \"browser/1_Storage.sol\",\n \"name\": \"Storage\",\n \"compilerVersion\": \"0.6.6+commit.6c089d02\",\n \"files\": {\n \"found\": [\n \"browser/1_Storage.sol\"\n ],\n \"missing\": []\n },\n \"status\": \"error\"\n }\n ],\n \"unused\": []\n}" + "body": "{\n \"result\": [\n {\n \"address\": \"0x656d0062eC89c940213E3F3170EA8b2add1c0143\",\n \"status\": \"perfect\"\n }\n ]\n}" } ] }, { - "name": "Add input files - without metadata", + "name": "verify - missing - do not fetch", "request": { "method": "POST", "header": [], "body": { "mode": "formdata", "formdata": [ + { + "key": "address", + "value": "0x656d0062eC89c940213E3F3170EA8b2add1c0143", + "type": "text" + }, + { + "key": "chain", + "value": "100", + "type": "text" + }, { "key": "files", "type": "file", - "src": "test/testcontracts/1_Storage/1_Storage.sol" + "src": "test/testcontracts/1_Storage/metadata.json" + } + ] + }, + "url": { + "raw": "{{host}}", + "host": [ + "{{host}}" + ] + } + }, + "response": [ + { + "name": "verify - missing - do not fetch", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "address", + "value": "0x656d0062eC89c940213E3F3170EA8b2add1c0143", + "type": "text" + }, + { + "key": "chain", + "value": "100", + "type": "text" + }, + { + "key": "files", + "type": "file", + "src": "test/testcontracts/1_Storage/metadata.json" + } + ] + }, + "url": { + "raw": "{{host}}", + "host": [ + "{{host}}" + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Access-Control-Allow-Origin", + "value": "*" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "75" + }, + { + "key": "ETag", + "value": "W/\"4b-91ydo9mIuZsJIBw/zbFBZMMd6nk\"" + }, + { + "key": "Set-Cookie", + "value": "sourcify_vid=s%3A1MUnH1uhcDyJytHrXoYrnU3pnYpHwg4k.ESJevK%2BKUpvwiC9MxqT%2FJTXaU%2BGKSIv3yYqJlJ4W6MA; Path=/; Expires=Wed, 20 Jan 2021 23:50:15 GMT; HttpOnly" + }, + { + "key": "Date", + "value": "Wed, 20 Jan 2021 11:50:15 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + } + ], + "cookie": [], + "body": "{\n \"error\": \"Invalid or missing sources in:\\nStorage (browser/1_Storage.sol)\"\n}" + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ] + }, + { + "name": "Verification API 2", + "item": [ + { + "name": "Add input files - success", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "files", + "type": "file", + "src": [ + "test/testcontracts/1_Storage/1_Storage.sol", + "test/testcontracts/1_Storage/metadata.json" + ] } ] }, @@ -226,7 +354,7 @@ }, "response": [ { - "name": "Add input files - without metadata", + "name": "Add input files - success", "originalRequest": { "method": "POST", "header": [], @@ -236,7 +364,10 @@ { "key": "files", "type": "file", - "src": "test/testcontracts/1_Storage/1_Storage.sol" + "src": [ + "test/testcontracts/1_Storage/1_Storage.sol", + "test/testcontracts/1_Storage/metadata.json" + ] } ] }, @@ -268,19 +399,19 @@ }, { "key": "Content-Length", - "value": "43" + "value": "287" }, { "key": "ETag", - "value": "W/\"2b-GRCkRQ7DAtUZ6oOVl+Q+eZi9siI\"" + "value": "W/\"11f-g6j9TfRz4+JLOVlzsIMF7fjowkQ\"" }, { "key": "Set-Cookie", - "value": "sourcify_vid=s%3Ai8fGEBmscCNl4g5Lr-55tEiTPRIFJZI3.i%2Bg%2B9doxHuunTqq5V6nPPsnkc5GIc8RUpuzVm2wFGqo; Path=/; Expires=Wed, 13 Jan 2021 05:40:11 GMT; HttpOnly" + "value": "sourcify_vid=s%3Ai8fGEBmscCNl4g5Lr-55tEiTPRIFJZI3.i%2Bg%2B9doxHuunTqq5V6nPPsnkc5GIc8RUpuzVm2wFGqo; Path=/; Expires=Wed, 13 Jan 2021 05:40:19 GMT; HttpOnly" }, { "key": "Date", - "value": "Tue, 12 Jan 2021 17:40:11 GMT" + "value": "Tue, 12 Jan 2021 17:40:19 GMT" }, { "key": "Connection", @@ -288,10 +419,64 @@ } ], "cookie": [], - "body": "{\n \"contracts\": [],\n \"unused\": [\n \"1_Storage.sol\"\n ]\n}" + "body": "{\n \"contracts\": [\n {\n \"verificationId\": \"0x3f67e9f57515bb1e7195c7c5af1eff630091567c0bb65ba3dece57a56da766fe\",\n \"compiledPath\": \"browser/1_Storage.sol\",\n \"name\": \"Storage\",\n \"compilerVersion\": \"0.6.6+commit.6c089d02\",\n \"files\": {\n \"found\": [\n \"browser/1_Storage.sol\"\n ],\n \"missing\": []\n },\n \"status\": \"error\"\n }\n ],\n \"unused\": []\n}" } ] }, + { + "name": "Add input files - source only", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "files", + "type": "file", + "src": "test/testcontracts/1_Storage/1_Storage.sol" + } + ] + }, + "url": { + "raw": "{{host}}/input-files", + "host": [ + "{{host}}" + ], + "path": [ + "input-files" + ] + } + }, + "response": [] + }, + { + "name": "Add input files - metadata only", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "files", + "type": "file", + "src": "test/testcontracts/1_Storage/metadata.json" + } + ] + }, + "url": { + "raw": "{{host}}/input-files", + "host": [ + "{{host}}" + ], + "path": [ + "input-files" + ] + } + }, + "response": [] + }, { "name": "Add input files - under wrong property", "request": { @@ -848,6 +1033,108 @@ } ] }, + { + "name": "Verify validated contracts - fetch missing source", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"contracts\": [\n {\n \"address\": \"0x656d0062eC89c940213E3F3170EA8b2add1c0143\",\n \"networkId\": \"100\",\n \"compilerVersion\": \"0.6.6+commit.6c089d02\",\n \"verificationId\": \"0x3f67e9f57515bb1e7195c7c5af1eff630091567c0bb65ba3dece57a56da766fe\"\n }\n ],\n \"fetch\": true\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/verify-validated", + "host": [ + "{{host}}" + ], + "path": [ + "verify-validated" + ] + } + }, + "response": [ + { + "name": "Verify validated contracts - fetch missing source", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"contracts\": [\n {\n \"address\": \"0x656d0062eC89c940213E3F3170EA8b2add1c0143\",\n \"networkId\": \"100\",\n \"compilerVersion\": \"0.6.6+commit.6c089d02\",\n \"verificationId\": \"0x3f67e9f57515bb1e7195c7c5af1eff630091567c0bb65ba3dece57a56da766fe\"\n }\n ],\n \"fetch\": true\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/verify-validated", + "host": [ + "{{host}}" + ], + "path": [ + "verify-validated" + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Access-Control-Allow-Origin", + "value": "*" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "375" + }, + { + "key": "ETag", + "value": "W/\"177-KLpUVJREY/f+juod/D1HYTPgH0w\"" + }, + { + "key": "Set-Cookie", + "value": "sourcify_vid=s%3AJc8SefWWpuHUIH6OAsjltayogQfAofJ2.DF%2FVQveyCQCjtNpOgza8CJ43Q2qSWCkjgepNb9hARmE; Path=/; Expires=Thu, 21 Jan 2021 00:21:19 GMT; HttpOnly" + }, + { + "key": "Date", + "value": "Wed, 20 Jan 2021 12:21:19 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + } + ], + "cookie": [], + "body": "{\n \"contracts\": [\n {\n \"verificationId\": \"0x3f67e9f57515bb1e7195c7c5af1eff630091567c0bb65ba3dece57a56da766fe\",\n \"compiledPath\": \"browser/1_Storage.sol\",\n \"name\": \"Storage\",\n \"compilerVersion\": \"0.6.6+commit.6c089d02\",\n \"address\": \"0x656d0062eC89c940213E3F3170EA8b2add1c0143\",\n \"networkId\": \"100\",\n \"files\": {\n \"found\": [\n \"browser/1_Storage.sol\"\n ],\n \"missing\": []\n },\n \"status\": \"perfect\"\n }\n ],\n \"unused\": [],\n \"fetch\": true\n}" + } + ] + }, { "name": "Restart session", "request": { diff --git a/docs/api/server/verification1/verify.md b/docs/api/server/verification1/verify.md index 47b3ecaf2..030ed8cf2 100644 --- a/docs/api/server/verification1/verify.md +++ b/docs/api/server/verification1/verify.md @@ -78,7 +78,7 @@ If using `application/json`, the files should be in an object under the key `fil **Condition** : Failed fetching missing files. OR Contract bytecode does not match deployed bytecode. -**Code** : `500 Internal Server Error` +**Code** : `400 Bad Request Error` **Content** : ```json diff --git a/docs/api/server/verification2/exchange-object.md b/docs/api/server/verification2/exchange-object.md index 7f52b5039..29a03c771 100644 --- a/docs/api/server/verification2/exchange-object.md +++ b/docs/api/server/verification2/exchange-object.md @@ -44,7 +44,8 @@ The object expected by the server from the client is a proper subset of (1) and "networkId": "100", "compilerVersion": "0.6.6+commit.6c089d02" } - ] + ], + "fetch": true // if true or "true", then considered true, all other values are treated as false } ``` - If the client does not know some of the properties yet, they may be omitted. diff --git a/docs/api/server/verification2/verify-validated.md b/docs/api/server/verification2/verify-validated.md index 8d6789704..65bded062 100644 --- a/docs/api/server/verification2/verify-validated.md +++ b/docs/api/server/verification2/verify-validated.md @@ -13,9 +13,9 @@ ## Responses -**Assumptions for all example responses (except the last one)** : -* There is one pending contract with all source files, but no `address` or `networkId`. -* Supplying the following minimum object (extra properties would be ignored): +**Assumptions for all example responses (except the last)** : +* There is one pending contract with 0 or 1 missing source file, but no `address` or `networkId`. +* Supplying the following minimum object: ```json { "contracts": [ @@ -29,7 +29,9 @@ } ``` -**Condition** : The provided contract `perfect`ly matches the one at the provided `networkId` and `address`. +**Conditions** : +- All the source files have been previously provided. +- The provided contract `perfect`ly matches the one at the provided `networkId` and `address`. **Code** : `200 OK` @@ -60,6 +62,41 @@ ### OR +**Conditions** : +- One source file is missing. +- Fetching requested by providing `fetch: true`, look [here](exchange-object.md) for more info. +- The provided contract `perfect`ly matches the one at the provided `networkId` and `address`. + +**Code** : `200 OK` + +**Content** : + +```json +{ + "contracts": [ + { + "verificationId": "0x3f67e9f57515bb1e7195c7c5af1eff630091567c0bb65ba3dece57a56da766fe", + "compiledPath": "browser/1_Storage.sol", + "name": "Storage", + "compilerVersion": "0.6.6+commit.6c089d02", + "address": "0x656d0062eC89c940213E3F3170EA8b2add1c0143", + "networkId": "100", + "files": { + "found": [ + "browser/1_Storage.sol" + ], + "missing": [] + }, + "status": "perfect" + } + ], + "unused": [], + "fetch": true +} +``` + +### OR + **Condition** : The contract at the provided `networkId` and `address` has already been verified at `2021-01-12T15:41:56.502Z`. **Code** : `200 OK` diff --git a/services/core/src/utils/CheckedContract.ts b/services/core/src/utils/CheckedContract.ts index 307370797..3a250ffbc 100644 --- a/services/core/src/utils/CheckedContract.ts +++ b/services/core/src/utils/CheckedContract.ts @@ -43,10 +43,15 @@ export class CheckedContract { name: string; /** Checks whether this contract is valid or not. + * This is a static method due to persistence issues. + * + * @param contract the contract to be checked + * @param ignoreMissing a flag indicating that missing sources should be ignored * @returns true if no sources are missing or are invalid (malformed); false otherwise */ - public static isValid(contract: CheckedContract): boolean { - return isEmpty(contract.missing) && isEmpty(contract.invalid); + public static isValid(contract: CheckedContract, ignoreMissing?: boolean): boolean { + return (isEmpty(contract.missing) || ignoreMissing) + && isEmpty(contract.invalid); } private sourceMapToStringMap(input: SourceMap) { @@ -196,65 +201,51 @@ export class CheckedContract { * * @param log log object */ - public async fetchMissing(log?: bunyan): Promise { + public static async fetchMissing(contract: CheckedContract, log?: bunyan): Promise { const retrieved: StringMap = {}; - for (const fileName in this.missing) { - if (!fileName.startsWith("@openzeppelin")) { - continue; - } - - const file = this.missing[fileName]; - const hash = this.missing[fileName].keccak256; - - let success = false; - for (const url of file.urls) { - if (url.startsWith(IPFS_PREFIX)) { - const ipfsCode = url.slice(IPFS_PREFIX.length); - const ipfsUrl = 'https://ipfs.infura.io:5001/api/v0/cat?arg='+ipfsCode; - const retrievedContent = await this.performFetch(ipfsUrl, hash, fileName, log); - if (retrievedContent) { - success = true; - retrieved[fileName] = retrievedContent; - break; + const missingFiles: string[] = []; + for (const fileName in contract.missing) { + const file = contract.missing[fileName]; + const hash = contract.missing[fileName].keccak256; + + let retrievedContent = null; + + const githubUrl = getGithubUrl(fileName); + if (githubUrl) { + retrievedContent = await performFetch(githubUrl, hash, fileName, log); + + } else { + for (const url of file.urls) { + if (url.startsWith(IPFS_PREFIX)) { + const ipfsCode = url.slice(IPFS_PREFIX.length); + const ipfsUrl = 'https://ipfs.infura.io:5001/api/v0/cat?arg='+ipfsCode; + retrievedContent = await performFetch(ipfsUrl, hash, fileName, log); + if (retrievedContent) { + break; + } } } } + + if (retrievedContent) { + retrieved[fileName] = retrievedContent; + } else { + missingFiles.push(fileName); + break; // makes an early exit + } } for (const fileName in retrieved) { - delete this.missing[fileName]; - this.solidity[fileName] = retrieved[fileName]; + delete contract.missing[fileName]; + contract.solidity[fileName] = retrieved[fileName]; } - const missingFiles = Object.keys(this.missing); if (missingFiles.length) { log.error({ loc: "[FETCH]", contractName: this.name, missingFiles }); - throw new Error(`Missing sources after fetching: ${missingFiles.join(", ")}`); - } - } - - private async performFetch(url: string, hash: string, fileName: string, log?: bunyan): Promise { - const infoObject = { loc: "[FETCH]", fileName, url }; - if (log) log.info(infoObject, "Fetch attempt"); - - const res = await fetch(url); - if (res.status === 200) { - const content = await res.text(); - if (Web3.utils.keccak256(content) !== hash) { - if (log) log.error(infoObject, "The calculated and the provided hash don't match."); - return null; - } - - if (log) log.info(infoObject, "Fetch successful!"); - return content; - - } else { - if (log) log.error(infoObject, "Fetch failed!"); - return null; + throw new Error(`Resource missing; unsuccessful fetching: ${missingFiles.join(", ")}`); } } - /** * Returns a message describing the errors encountered while validating the metadata. * Does not include a trailing newline. @@ -264,4 +255,49 @@ export class CheckedContract { public getInfo() { return CheckedContract.isValid(this) ? this.composeSuccessMessage() : this.composeErrorMessage(); } +} + +/** + * Performs fetch and compares with the hash provided. + * + * @param url the url to be used as the file source + * @param hash the hash of the file to be fetched; used for later comparison + * @param fileName the name of the file; used for logging + * @param log whether or not to log + * @returns the fetched file if found; null otherwise + */ +async function performFetch(url: string, hash: string, fileName: string, log?: bunyan): Promise { + const infoObject = { loc: "[FETCH]", fileName, url }; + if (log) log.info(infoObject, "Fetch attempt"); + + const res = await fetch(url); + if (res.status === 200) { + const content = await res.text(); + if (Web3.utils.keccak256(content) !== hash) { + if (log) log.error(infoObject, "The calculated and the provided hash don't match."); + return null; + } + + if (log) log.info(infoObject, "Fetch successful!"); + return content; + + } else { + if (log) log.error(infoObject, "Fetch failed!"); + return null; + } +} + +/** + * Makes a GitHub-compatible url out of the provided url, if possible. + * + * @param url + * @returns a GitHub-compatible url if possible; null otherwise + */ +function getGithubUrl(url: string): string { + if (!url.includes("github.com")) { + return null; + } + return url + .replace("github.com", "raw.githubusercontent.com") + .replace("/blob/", "/"); } \ No newline at end of file diff --git a/services/core/src/utils/types.ts b/services/core/src/utils/types.ts index b380283ae..72fd8bf3e 100644 --- a/services/core/src/utils/types.ts +++ b/services/core/src/utils/types.ts @@ -12,6 +12,7 @@ export interface InputData { addresses: string[], contract?: CheckedContract, bytecode?: string; + fetchMissing?: boolean; } export interface CompilationSettings { diff --git a/services/verification/src/services/Injector.ts b/services/verification/src/services/Injector.ts index b4d45a794..4eb3b82a9 100644 --- a/services/verification/src/services/Injector.ts +++ b/services/verification/src/services/Injector.ts @@ -195,7 +195,7 @@ export class Injector { * @return {Promise} address & status of successfully verified contract */ public async inject(inputData: InputData): Promise { - const { chain, addresses, contract } = inputData; + const { chain, addresses, contract, fetchMissing } = inputData; this.validateAddresses(addresses); this.validateChain(chain); @@ -209,8 +209,8 @@ export class Injector { if (!CheckedContract.isValid(contract)) { // eslint-disable-next-line no-useless-catch - try { - await contract.fetchMissing(this.log); + if (fetchMissing) try { + await CheckedContract.fetchMissing(contract, this.log); } catch(err) { throw err; } diff --git a/services/verification/src/utils.ts b/services/verification/src/utils.ts index 1866ad62a..501be436e 100644 --- a/services/verification/src/utils.ts +++ b/services/verification/src/utils.ts @@ -6,6 +6,10 @@ import fs from 'fs'; // eslint-disable-next-line @typescript-eslint/no-var-requires const solc = require('solc'); import {spawnSync} from 'child_process'; +import { StatusCodes } from 'http-status-codes'; +import { mkdirpSync } from 'fs-extra'; + +const GITHUB_SOLC_REPO = "https://github.com/ethereum/solc-bin/raw/gh-pages/linux-amd64/"; export interface RecompilationResult { bytecode: string, @@ -102,7 +106,8 @@ export async function recompile( } /** - * Searches for a solc: first for a local executable version, then using the getSolc function. + * Searches for a solc: first for a local executable version, then from GitHub + * and then using the getSolc function. * Once the compiler is retrieved, it is used, and the stringified solc output is returned. * * @param version the version of solc to be used for compilation @@ -113,9 +118,14 @@ export async function recompile( async function useCompiler(version: string, input: any, log: {info: any, error: any}) { const inputStringified = JSON.stringify(input); const repoPath = process.env.SOLC_REPO || "solc-repo"; - const solcPath = Path.join(repoPath, `solc-linux-amd64-v${version}`); + const fileName = `solc-linux-amd64-v${version}`; + const solcPath = Path.join(repoPath, fileName); let compiled: string = null; + if (!fs.existsSync(solcPath)) { + await fetchSolcFromGitHub(solcPath, version, fileName, log); + } + if (fs.existsSync(solcPath)) { log.info({loc: "[RECOMPILE]", version, solcPath}, "Compiling with external executable"); const shellOutputBuffer = spawnSync(solcPath, ["--standard-json"], {input: inputStringified}); @@ -128,6 +138,23 @@ async function useCompiler(version: string, input: any, log: {info: any, error: return compiled; } +async function fetchSolcFromGitHub(solcPath: string, version: string, fileName: string, log: {info: any, error: any}) { + const githubSolcURI = GITHUB_SOLC_REPO + encodeURIComponent(fileName); + const logObject = {loc: "[RECOMPILE]", version, githubSolcURI}; + log.info(logObject, "Fetching executable solc from GitHub"); + + const res = await fetch(githubSolcURI); + if (res.status === StatusCodes.OK) { + log.info(logObject, "Successfully fetched executable solc from GitHub"); + mkdirpSync(Path.dirname(solcPath)); + const buffer = await res.buffer(); + fs.writeFileSync(solcPath, buffer, { mode: 0o755 }); + + } else { + log.error(logObject, "Failed fetching executable solc from GitHub"); + } +} + /** * Removes post-fixed metadata from a bytecode string * (for partial bytecode match comparisons ) diff --git a/src/server/controllers/VerificationController-util.ts b/src/server/controllers/VerificationController-util.ts index c8deafa16..4a1b8db49 100644 --- a/src/server/controllers/VerificationController-util.ts +++ b/src/server/controllers/VerificationController-util.ts @@ -39,7 +39,8 @@ export type SessionMaps = { export type MySession = Session & SessionMaps & { - unusedSources: string[] + unusedSources: string[], + fetch?: boolean }; export type MyRequest = @@ -59,9 +60,9 @@ export type SendableContract = verificationId?: string } -export function isVerifiable(contractWrapper: ContractWrapper) { +export function isVerifiable(contractWrapper: ContractWrapper, ignoreMissing?: boolean) { const contract = contractWrapper.contract; - return isEmpty(contract.missing) + return (isEmpty(contract.missing) || ignoreMissing) && isEmpty(contract.invalid) && Boolean(contractWrapper.compilerVersion) && Boolean(contractWrapper.address) @@ -103,7 +104,7 @@ export function getSessionJSON(session: MySession) { } const unused = session.unusedSources || []; - return { contracts, unused }; + return { contracts, unused, fetch: session.fetch }; } export function generateId(obj: any): string { diff --git a/src/server/controllers/VerificationController.ts b/src/server/controllers/VerificationController.ts index 9bb83715e..c703a93d5 100644 --- a/src/server/controllers/VerificationController.ts +++ b/src/server/controllers/VerificationController.ts @@ -62,6 +62,11 @@ export default class VerificationController extends BaseController implements IC return validChainIds; } + private stringifyInvalidAndMissing(contract: CheckedContract) { + const errors = Object.keys(contract.invalid).concat(Object.keys(contract.missing)); + return `${contract.name} (${errors.join(", ")})`; + } + private legacyVerifyEndpoint = async (origReq: Request, res: Response): Promise => { const req = (origReq as MyRequest); this.validateRequest(req); @@ -86,11 +91,12 @@ export default class VerificationController extends BaseController implements IC throw new BadRequestError(error.message); } + const ignoreMissing = req.body.fetch; const errors = validatedContracts - .filter(contract => Object.keys(contract.invalid).length) - .map(contract => `${contract.name} ${Object(contract.invalid).keys()}`); + .filter(contract => !CheckedContract.isValid(contract, ignoreMissing)) + .map(this.stringifyInvalidAndMissing); if (errors.length) { - throw new BadRequestError("Errors in:\n" + errors.join("\n"), false); + throw new BadRequestError("Invalid or missing sources in:\n" + errors.join("\n"), false); } if (validatedContracts.length !== 1) { @@ -103,7 +109,9 @@ export default class VerificationController extends BaseController implements IC if (!contract.compilerVersion) { throw new BadRequestError("Metadata file not specifying a compiler version."); } - const inputData: InputData = { contract, addresses: req.addresses, chain: req.chain }; + + const fetchMissing = req.body.fetch; + const inputData: InputData = { contract, addresses: req.addresses, chain: req.chain, fetchMissing }; const resultPromise = this.verificationService.inject(inputData, config.localchain.url); resultPromise.then(result => { @@ -175,6 +183,7 @@ export default class VerificationController extends BaseController implements IC const receivedContracts: SendableContract[] = req.body.contracts; + const ignoreMissing = req.body.fetch; const verifiable: ContractWrapperMap = {}; for (const receivedContract of receivedContracts) { const id = receivedContract.verificationId; @@ -183,23 +192,25 @@ export default class VerificationController extends BaseController implements IC contractWrapper.address = receivedContract.address; contractWrapper.networkId = receivedContract.networkId; contractWrapper.compilerVersion = receivedContract.compilerVersion; - if (isVerifiable(contractWrapper)) { + contractWrapper.contract.metadata.compiler.version = receivedContract.compilerVersion; + if (isVerifiable(contractWrapper, ignoreMissing)) { verifiable[id] = contractWrapper; } } } - await this.verifyValidated(verifiable); + await this.verifyValidated(verifiable, req.body.fetch); res.send(getSessionJSON(session)); } - private async verifyValidated(contractWrappers: ContractWrapperMap): Promise { + private async verifyValidated(contractWrappers: ContractWrapperMap, fetchMissing?: boolean): Promise { for (const id in contractWrappers) { const contractWrapper = contractWrappers[id]; - if (!isVerifiable(contractWrapper)) { + const ignoreMissing = fetchMissing; + if (!isVerifiable(contractWrapper, ignoreMissing)) { continue; } - const inputData: InputData = { addresses: [contractWrapper.address], chain: contractWrapper.networkId, contract: contractWrapper.contract }; + const inputData: InputData = { addresses: [contractWrapper.address], chain: contractWrapper.networkId, contract: contractWrapper.contract, fetchMissing }; const found = await this.verificationService.findByAddress(contractWrapper.address, contractWrapper.networkId, config.repository.path); let match: Match; @@ -283,7 +294,7 @@ export default class VerificationController extends BaseController implements IC this.saveFiles(pathContents, session); this.validateContracts(session); - await this.verifyValidated(session.contractWrappers); + await this.verifyValidated(session.contractWrappers, req.body.fetch); res.send(getSessionJSON(session)); } @@ -322,11 +333,18 @@ export default class VerificationController extends BaseController implements IC } } + private fetchSanitizer = body("fetch").customSanitizer((f, { req }) => { + const sanitized = f === true || f === "true"; + req.session.fetch = sanitized; + return sanitized; + }); + registerRoutes = (): Router => { this.router.route(['/', '/verify']) .post( body("address").exists().bail().custom((address, { req }) => req.addresses = this.validateAddresses(address)), body("chain").exists().bail().custom((chain, { req }) => req.chain = getChainId(chain)), + this.fetchSanitizer, this.safeHandler(this.legacyVerifyEndpoint) ); @@ -347,7 +365,11 @@ export default class VerificationController extends BaseController implements IC .post(this.safeHandler(this.restartSessionEndpoint)); this.router.route('/verify-validated') - .post(body("contracts").isArray(), this.safeHandler(this.verifyValidatedEndpoint)); + .post( + body("contracts").isArray(), + this.fetchSanitizer, + this.safeHandler(this.verifyValidatedEndpoint) + ); return this.router; } diff --git a/test/server.js b/test/server.js index 56f5b0376..b965ad650 100644 --- a/test/server.js +++ b/test/server.js @@ -11,6 +11,7 @@ const fs = require("fs"); const rimraf = require("rimraf"); const path = require("path"); const MAX_INPUT_SIZE = require("../dist/server/controllers/VerificationController").default.MAX_INPUT_SIZE; +const StatusCodes = require('http-status-codes').StatusCodes; chai.use(chaiHttp); const EXTENDED_TIME = 15000; // 15 seconds @@ -39,7 +40,7 @@ describe("Server", async () => { const assertError = (err, res, field) => { chai.expect(err).to.be.null; - chai.expect(res.status).to.equal(400); + chai.expect(res.status).to.equal(StatusCodes.BAD_REQUEST); chai.expect(res.body.message.startsWith("Validation Error")); chai.expect(res.body.errors).to.be.an("array"); chai.expect(res.body.errors).to.have.a.lengthOf(1); @@ -69,7 +70,7 @@ describe("Server", async () => { const assertStatus = (err, res, expectedStatus, done) => { chai.expect(err).to.be.null; - chai.expect(res.status).to.equal(200); + chai.expect(res.status).to.equal(StatusCodes.OK); const resultArray = res.body; chai.expect(resultArray).to.have.a.lengthOf(1); const result = resultArray[0]; @@ -81,7 +82,7 @@ describe("Server", async () => { it("should return false for previously unverified contract", (done) => { chai.request(server.app) .get("/check-by-Addresses") - .query({ chainIds: 100, addresses: contractAddress }) + .query({ chainIds: contractChain, addresses: contractAddress }) .end((err, res) => assertStatus(err, res, "false", done)); }).timeout(EXTENDED_TIME); @@ -98,7 +99,7 @@ describe("Server", async () => { it("should return true for previously verified contract", (done) => { chai.request(server.app) .get("/check-by-addresses") - .query({ chainIds: 100, addresses: contractAddress }) + .query({ chainIds: contractChain, addresses: contractAddress }) .end((err, res) => { assertStatus(err, res, "false"); chai.request(server.app).post("/") @@ -108,11 +109,11 @@ describe("Server", async () => { .attach("files", sourceBuffer) .end((err, res) => { chai.expect(err).to.be.null; - chai.expect(res.status).to.equal(200); + chai.expect(res.status).to.equal(StatusCodes.OK); chai.request(server.app) .get("/check-by-addresses") - .query({ chainIds: 100, addresses: contractAddress }) + .query({ chainIds: contractChain, addresses: contractAddress }) .end((err, res) => assertStatus(err, res, "perfect", done)); }); }); @@ -143,7 +144,7 @@ describe("Server", async () => { const assertions = (err, res, done) => { chai.expect(err).to.be.null; - chai.expect(res.status).to.equal(200); + chai.expect(res.status).to.equal(StatusCodes.OK); chai.expect(res.body).to.haveOwnProperty("result"); const resultArr = res.body.result; chai.expect(resultArr).to.have.a.lengthOf(1); @@ -191,19 +192,56 @@ describe("Server", async () => { .end((err, res) => assertions(err, res, done)); }).timeout(EXTENDED_TIME); - it("should return Internal Server Error for missing file", (done) => { + const assertMissingFile = (err, res) => { + chai.expect(err).to.be.null; + chai.expect(res.body).to.haveOwnProperty("error"); + const errorMessage = res.body.error.toLowerCase(); + chai.expect(res.status).to.equal(StatusCodes.BAD_REQUEST); + chai.expect(errorMessage).to.include("missing"); + chai.expect(errorMessage).to.include("Storage".toLowerCase()); + } + + it("should return Bad Request Error for missing file", (done) => { chai.request(server.app) .post("/") .field("address", contractAddress) .field("chain", contractChain) .attach("files", metadataBuffer, "metadata.json") .end((err, res) => { - chai.expect(err).to.be.null; - chai.expect(res.body).to.haveOwnProperty("error"); - const errorMessage = res.body.error.toLowerCase(); - chai.expect(res.status).to.equal(500); - chai.expect(errorMessage).to.include("missing"); - chai.expect(errorMessage).to.include("1_Storage.sol".toLowerCase()); + assertMissingFile(err, res); + done(); + }); + }).timeout(EXTENDED_TIME); + + it("should fetch a missing file that is accessible via ipfs (using fetch=true)", (done) => { + chai.request(server.app) + .post("/") + .field("address", contractAddress) + .field("chain", contractChain) + .field("fetch", true) + .attach("files", metadataBuffer, "metadata.json") + .end((err, res) => assertions(err, res, done)); + }).timeout(EXTENDED_TIME); + + it("should fetch a missing file that is accessible via ipfs (using fetch=\"true\")", (done) => { + chai.request(server.app) + .post("/") + .field("address", contractAddress) + .field("chain", contractChain) + .field("fetch", "true") + .attach("files", metadataBuffer, "metadata.json") + .end((err, res) => assertions(err, res, done)); + }).timeout(EXTENDED_TIME); + + it("should not fetch a missing file if the 'fetch' property is not true or \"true\"", (done) => { + chai.request(server.app) + .post("/") + .field("address", contractAddress) + .field("chain", contractChain) + .field("fetch", 1) + .attach("files", metadataBuffer, "metadata.json") + .end((err, res) => { + assertMissingFile(err, res); done(); }); }).timeout(EXTENDED_TIME); @@ -216,32 +254,53 @@ describe("Server", async () => { .end((err, res) => { chai.expect(err).to.be.null; chai.expect(res.body).to.haveOwnProperty("error"); - chai.expect(res.status).to.equal(400); + chai.expect(res.status).to.equal(StatusCodes.BAD_REQUEST); chai.expect(res.body.error).to.equal("There are currently no pending contracts.") done(); }); }).timeout(EXTENDED_TIME); + const assertOnlyAddressAndChainMissing = (res) => { + chai.expect(res.status).to.equal(StatusCodes.OK); + const contracts = res.body.contracts; + chai.expect(contracts).to.have.a.lengthOf(1); + const contract = contracts[0]; + chai.expect(contract.status).to.equal("error"); + chai.expect(contract.files.missing).to.deep.equal([]); + chai.expect(contract.files.found).to.deep.equal(["browser/1_Storage.sol"]); + chai.expect(res.body.unused).to.be.empty; + return contracts; + }; + + it("should accept file upload in JSON format", (done) => { + chai.request(server.app) + .post("/input-files") + .send({ + files: { + "metadata.json": metadataBuffer.toString(), + "1_Storage.sol": sourceBuffer.toString() + } + }).then(res => { + assertOnlyAddressAndChainMissing(res); + done(); + }) + }).timeout(EXTENDED_TIME); + it("should not verify after addition of metadata+source, but should after providing address+networkId", (done) => { const agent = chai.request.agent(server.app); agent.post("/input-files") .attach("files", sourceBuffer, "1_Storage.sol") .attach("files", metadataBuffer, "metadata.json") .then(res => { - chai.expect(res.status).to.equal(200); - const contracts = res.body.contracts; - chai.expect(contracts).to.have.a.lengthOf(1); - const contract = contracts[0]; - chai.expect(contract.status).to.equal("error"); - chai.expect(res.body.unused).to.be.empty; - contract.address = contractAddress; - contract.networkId = contractChain; + const contracts = assertOnlyAddressAndChainMissing(res); + contracts[0].address = contractAddress; + contracts[0].networkId = contractChain; agent.post("/verify-validated") .send({ contracts }) .end((err, res) => { chai.expect(err).to.be.null; - chai.expect(res.status).to.equal(200); + chai.expect(res.status).to.equal(StatusCodes.OK); const contracts = res.body.contracts; chai.expect(contracts).to.have.a.lengthOf(1); const contract = contracts[0]; @@ -255,7 +314,7 @@ describe("Server", async () => { const assertAfterMetadataUpload = (err, res) => { chai.expect(err).to.be.null; - chai.expect(res.status).to.equal(200); + chai.expect(res.status).to.equal(StatusCodes.OK); chai.expect(res.body.unused).to.be.empty; const contracts = res.body.contracts; @@ -278,7 +337,7 @@ describe("Server", async () => { .attach("files", sourceBuffer, "1_Storage.sol") .end((err, res) => { chai.expect(err).to.be.null; - chai.expect(res.status).to.equal(200); + chai.expect(res.status).to.equal(StatusCodes.OK); chai.expect(res.body.unused).to.deep.equal(["1_Storage.sol"]); chai.expect(res.body.contracts).to.be.empty; @@ -289,7 +348,7 @@ describe("Server", async () => { const assertAllFound = (err, res, finalStatus) => { chai.expect(err).to.be.null; - chai.expect(res.status).to.equal(200); + chai.expect(res.status).to.equal(StatusCodes.OK); chai.expect(res.body.unused).to.be.empty; const contracts = res.body.contracts; @@ -333,7 +392,7 @@ describe("Server", async () => { agent.post("/input-files") .attach("files", Buffer.from(file)) .then(res => { - chai.expect(res.status).to.equal(200); + chai.expect(res.status).to.equal(StatusCodes.OK); agent.post("/input-files") .attach("files", Buffer.from("a")) @@ -343,12 +402,12 @@ describe("Server", async () => { agent.post("/restart-session") .then(res => { - chai.expect(res.status).to.equal(200); + chai.expect(res.status).to.equal(StatusCodes.OK); agent.post("/input-files") .attach("files", Buffer.from("a")) .then(res => { - chai.expect(res.status).to.equal(200); + chai.expect(res.status).to.equal(StatusCodes.OK); done(); }); }); @@ -357,7 +416,7 @@ describe("Server", async () => { }).timeout(EXTENDED_TIME); const assertSingleContractStatus = (res, expectedStatus, shouldHaveTimestamp) => { - chai.expect(res.status).to.equal(200); + chai.expect(res.status).to.equal(StatusCodes.OK); chai.expect(res.body).to.haveOwnProperty("contracts"); const contracts = res.body.contracts; chai.expect(contracts).to.have.a.lengthOf(1); @@ -397,5 +456,30 @@ describe("Server", async () => { }); }); }).timeout(EXTENDED_TIME); + + it("should verify after being told to fetch", (done) => { + const agent = chai.request.agent(server.app); + agent.post("/input-files") + .attach("files", metadataBuffer) + .then(res => { + const contracts = assertSingleContractStatus(res, "error"); + contracts[0].address = contractAddress; + contracts[0].networkId = contractChain; + + agent.post("/verify-validated") + .send({ contracts }) + .then(res => { + assertSingleContractStatus(res, "error"); + + agent.post("/verify-validated") + .send({ contracts, fetch: true }) + .then(res => { + assertSingleContractStatus(res, "perfect"); + chai.expect(res.body.fetch); + done(); + }) + }) + }) + }).timeout(EXTENDED_TIME); }); }); \ No newline at end of file