Merge pull request #50 from Web3-API/feat/standardize-web3api-manifest

Feat/standardize web3api manifest

demos/simple-storage/protocol/recipes/constants.json

@@ -1,3 +1,3 @@
 {
-  "SimpleStorageAddr": "0xe78A0F7E598Cc8b0Bb87894B0F60dD2a88d6a8Ab"
+  "SimpleStorageAddr": "0xfAa5Bd59B7Be0b06705D8EaED0636288279860b4"
 }

demos/simple-storage/protocol/src/subgraph/subgraph.yaml

@@ -8,7 +8,7 @@ dataSources:
     name: SimpleStorage
     network: private
     source:
-      address: '0xe78A0F7E598Cc8b0Bb87894B0F60dD2a88d6a8Ab'
+      address: '0xfAa5Bd59B7Be0b06705D8EaED0636288279860b4'
       abi: SimpleStorage
     mapping:
       kind: ethereum/events

demos/simple-storage/protocol/web3api.yaml

@@ -1,3 +1,4 @@
+format: 0.0.1-alpha.1
 description: SimpleStorage Web3API Example
 repository: https://github.com/web3-api/prototype
 mutation:

packages/cli/src/lib/Compiler.ts

@@ -1,16 +1,14 @@
 import { Web3API } from "./Web3API";
-import { Manifest } from "./Manifest";
 import { runGraphCLI } from "./cli/graph-cli";
 import { displayPath } from "./helpers/path";
 import { step, withSpinner } from "./helpers/spinner";
 
+import { Manifest } from "@web3api/client-js";
+import * as asc from "assemblyscript/cli/asc";
 import fs from "fs";
 import path from "path";
-import YAML from "js-yaml";
-import * as asc from "assemblyscript/cli/asc";
 
 const fsExtra = require("fs-extra");
-const spawn = require("spawn-command");
 const toolbox = require("gluegun/toolbox");
 
 export interface ICompilerConfig {

packages/cli/src/lib/Web3API.ts

@@ -1,21 +1,18 @@
-import { Manifest } from "./Manifest";
-
 import YAML from "js-yaml";
 import fs from "fs";
+import { sanitizeAndUpgrade, Manifest } from "@web3api/client-js";
 
 export class Web3API {
   public static load(manifestPath: string): Manifest {
-    const manifest = YAML.safeLoad(
+    let manifest = YAML.safeLoad(
       fs.readFileSync(manifestPath, "utf-8")
     ) as Manifest | undefined;
 
     if (!manifest) {
       throw Error(`Unable to parse manifest: ${manifestPath}`);
     }
-
-    // TODO: add validation
-    // - validate manifest structure
-    // - ensure everything exists
+
+    manifest = sanitizeAndUpgrade(manifest)
 
     return manifest;
   }

packages/client-js/package.json

@@ -4,7 +4,8 @@
   "license": "MIT",
   "main": "build/index.js",
   "scripts": {
-    "build": "rimraf build && tsc",
+    "build": "rimraf build && yarn build:manifest && tsc",
+    "build:manifest": "node ./scripts/generateManifest.js",
     "test": "WORKER_ENV=test jest --passWithNoTests --runInBand --verbose",
     "test:watch": "WORKER_ENV=test jest --watch --passWithNoTests --verbose",
     "test:ci": "yarn test:env:up && yarn test && yarn test:env:down",
@@ -22,14 +23,19 @@
     "graphql-tag": "2.10.4",
     "ipfs-http-client": "45.0.0",
     "is-ipfs": "1.0.3",
-    "js-yaml": "3.14.0"
+    "js-yaml": "3.14.0",
+    "jsonschema": "1.3.0",
+    "semver": "7.3.2"
   },
   "devDependencies": {
     "@types/jest": "26.0.8",
     "@types/js-yaml": "3.11.1",
+    "@types/semver":"^7.3.4",
     "@web3api/cli": "0.0.1-alpha.1",
+    "@web3api/manifest-schema": "0.0.1-alpha.1",
     "axios": "0.19.2",
     "jest": "26.2.2",
+    "json-schema-to-typescript": "9.1.1",
     "rimraf": "3.0.2",
     "spawn-command": "0.0.2-1",
     "ts-jest": "26.1.4",

packages/client-js/scripts/generateManifest.js

@@ -0,0 +1,20 @@
+const package = require("../package.json");
+const schemaVersion = package.devDependencies["@web3api/manifest-schema"];
+const schema = require(`@web3api/manifest-schema/formats/${schemaVersion}`);
+const { compile } = require("json-schema-to-typescript");
+const { writeFileSync } = require("fs");
+
+const generateManifest = () => {
+  try {
+    const manifest = schema["manifest"];
+    compile(manifest, "Web3API").then((file) => {
+      const manifestPath =
+        __dirname + `/../src/manifest/formats/${package.version}.ts`;
+      writeFileSync(manifestPath, file);
+    });
+  } catch (e) {
+    console.log("Error generating the Manifest file: ", e);
+  }
+};
+
+generateManifest();

packages/client-js/src/Web3API.ts

@@ -1,3 +1,12 @@
+import { compare } from "semver";
+import {
+  buildSchema,
+  execute,
+  GraphQLSchema,
+  GraphQLObjectType
+} from "graphql";
+import YAML from "js-yaml";
+
 import {
   Ethereum,
   IPFS,
@@ -7,19 +16,11 @@ import { getHostImports } from "./host";
 import { isPromise } from "./lib/async";
 import {
   Query,
-  QueryResult,
-  Manifest,
-  ModulePath
+  QueryResult
 } from "./lib/types";
 import { WasmWorker } from "./lib/wasm-worker";
-
-import {
-  buildSchema,
-  execute,
-  GraphQLSchema,
-  GraphQLObjectType
-} from "graphql";
-import YAML from "js-yaml";
+import { AnyManifest, Manifest } from "./manifest/formats"
+import { ManifestFormats, upgradeManifest, latestFormat } from "./manifest";
 
 export interface IPortals {
   ipfs: IPFS;
@@ -102,14 +103,19 @@ export class Web3API {
       const { name, depth, type, path } = file;
 
       if (depth === 1 && type === "file" &&
-         (name === "web3api.yaml" || name === "web3apy.yml")) {
+         (name === "web3api.yaml" || name === "web3api.yml")) {
         const manifestStr = await portals.ipfs.catToString(path);
         this._manifest = YAML.safeLoad(manifestStr) as Manifest | undefined;
 
         if (this._manifest === undefined) {
           throw Error(`Unable to parse web3api.yaml\n${manifestStr}`);
         }
 
+        const currentManifest: AnyManifest = this._manifest;
+        if (compare(latestFormat, currentManifest.format) === -1) {
+          this._manifest = upgradeManifest(currentManifest, latestFormat as ManifestFormats);
+        }
+
         return this._manifest;
       }
     }
@@ -153,7 +159,7 @@ export class Web3API {
       }
 
       this._addResolvers(
-        manifest.mutation.module,
+        manifest.mutation.module.file,
         mutationType
       );
     }
@@ -164,7 +170,7 @@ export class Web3API {
       }
 
       this._addResolvers(
-        manifest.query.module,
+        manifest.query.module.file,
         queryType
       );
     }
@@ -228,7 +234,7 @@ export class Web3API {
     }*/
   }
 
-  private _addResolvers(module: ModulePath, schemaType: GraphQLObjectType<any, any>) {
+  private _addResolvers(modulePath: string, schemaType: GraphQLObjectType<any, any>) {
     const fields = schemaType.getFields();
     const fieldNames = Object.keys(fields);
 
@@ -240,7 +246,7 @@ export class Web3API {
 
         // Load the WASM source
         const wasm = await portals.ipfs.catToBuffer(
-          `${this._cid}/${module.file}`
+          `${this._cid}/${modulePath}`
         );
 
         // Instantiate it

packages/client-js/src/__tests__/Validator.spec.ts

@@ -0,0 +1,111 @@
+import fs from "fs";
+import YAML from "js-yaml";
+
+import {
+  sanitizeAndUpgrade,
+  upgradeManifest,
+  ManifestFormats,
+} from "../manifest";
+import { Manifest } from "../manifest/formats/0.0.1-alpha.1";
+
+describe("Validate web3api manifest ", () => {
+  it("Should throw file string does not exist error ", async () => {
+    const manifestPath =
+      __dirname + "/manifest/validator/file-string-malformed/web3api.yml";
+    const manifest = YAML.safeLoad(fs.readFileSync(manifestPath, "utf-8"));
+
+    expect(() => sanitizeAndUpgrade(manifest as Manifest)).toThrowError(
+      /is not a valid file path. Please use unix style relative paths./
+    );
+  });
+  it("Should throw incorrect version format error ", async () => {
+    const manifestPath =
+      __dirname + "/manifest/validator/incorrect-version-format/web3api.yml";
+    const manifest = YAML.safeLoad(fs.readFileSync(manifestPath, "utf-8"));
+
+    expect(() => sanitizeAndUpgrade(manifest as Manifest)).toThrowError(
+      /The manifest's format is not correct./
+    );
+  });
+  it("Should throw not accepted field error ", async () => {
+    const manifestPath =
+      __dirname + "/manifest/validator/not-accepted-field/web3api.yml";
+    const manifest = YAML.safeLoad(fs.readFileSync(manifestPath, "utf-8"));
+
+    expect(() => sanitizeAndUpgrade(manifest as Manifest)).toThrowError(
+      /is not accepted in the schema/
+    );
+  });
+  it("Should throw required field missing error ", async () => {
+    const manifestPath =
+      __dirname + "/manifest/validator/required-field-missing/web3api.yml";
+    const manifest = YAML.safeLoad(fs.readFileSync(manifestPath, "utf-8"));
+    expect(() => sanitizeAndUpgrade(manifest as Manifest)).toThrowError(
+      /Missing field:/
+    );
+  });
+  it("Should throw wrong type errpr ", async () => {
+    const manifestPath =
+      __dirname + "/manifest/validator/wrong-type/web3api.yml";
+    const manifest = YAML.safeLoad(fs.readFileSync(manifestPath, "utf-8"));
+
+    expect(() => sanitizeAndUpgrade(manifest as Manifest)).toThrowError(
+      /has a type error: /
+    );
+  });
+});
+
+describe("Manifest migration ", () => {
+  it("Should upgrade 0.0.1-alpha.1 to 0.0.1-alpha.2 ", () => {
+    const manifestPath =
+      __dirname + "/manifest/migrator/format-0.0.1-alpha.1/web3api.yml";
+    const newManifestPath =
+      __dirname + "/manifest/migrator/format-0.0.1-alpha.2/web3api.yml";
+    const manifest = YAML.safeLoad(
+      fs.readFileSync(manifestPath, "utf-8")
+    ) as Manifest;
+
+    const manifestGenerated = upgradeManifest(
+      manifest,
+      "0.0.1-alpha.2" as ManifestFormats
+    );
+    const newManifest = YAML.safeLoad(
+      fs.readFileSync(newManifestPath, "utf-8")
+    );
+
+    expect(manifestGenerated).toEqual(newManifest);
+  });
+
+  it("Should throw error because is unrecognized format ", () => {
+    const manifestPath =
+      __dirname + "/manifest/migrator/unrecognized-format/web3api.yml";
+    const manifest = YAML.safeLoad(
+      fs.readFileSync(manifestPath, "utf-8")
+    ) as Manifest;
+    expect(() =>
+      upgradeManifest(manifest, "0.0.1-alpha.2" as ManifestFormats)
+    ).toThrowError(/Unrecognized manifest format/);
+  });
+
+  it("Should throw error because the from format does not have a migrator", () => {
+    const manifestPath =
+      __dirname + "/manifest/migrator/format-0.0.1-alpha.2/web3api.yml";
+    const manifest = YAML.safeLoad(
+      fs.readFileSync(manifestPath, "utf-8")
+    ) as Manifest;
+    expect(() =>
+      upgradeManifest(manifest, "0.0.1-alpha.3" as ManifestFormats)
+    ).toThrowError(/From format 0.0.1-alpha.2 migrator does not exists/);
+  });
+
+  it("Should throw error because to format does not exists in the migrator", () => {
+    const manifestPath =
+      __dirname + "/manifest/migrator/format-0.0.1-alpha.1/web3api.yml";
+    const manifest = YAML.safeLoad(
+      fs.readFileSync(manifestPath, "utf-8")
+    ) as Manifest;
+    expect(() =>
+      upgradeManifest(manifest, "0.0.1-alpha.3" as ManifestFormats)
+    ).toThrowError(/Format to update 0.0.1-alpha.3 is not available in migrator of format 0.0.1-alpha.1/);
+  });
+});

packages/client-js/src/__tests__/apis/eth-get-put-string/web3api.yaml

@@ -1,3 +1,4 @@
+format: 0.0.1-alpha.1
 mutation:
   schema:
     file: ./src/mutation/schema.graphql

packages/client-js/src/__tests__/apis/ipfs-get-put-string/web3api.yaml

@@ -1,3 +1,4 @@
+format: 0.0.1-alpha.1
 mutation:
   schema:
     file: ./src/mutation/schema.graphql

packages/client-js/src/__tests__/apis/subgraph-query/web3api.yaml

@@ -1,3 +1,4 @@
+format: 0.0.1-alpha.1
 mutation:
   schema:
     file: ./src/mutation/schema.graphql

packages/client-js/src/__tests__/manifest/migrator/format-0.0.1-alpha.1/web3api.yml

@@ -0,0 +1,17 @@
+description: Awesome Web3API project
+format: "0.0.1-alpha.1"
+mutation:
+  schema:
+    file: ../mutation/schema.graphql
+  module:
+    file: ../mutation/index.ts
+    language: wasm/assemblyscript
+query:
+  schema:
+    file: ../query/schema.graphql
+  module:
+    file: ../query/index.ts
+    language: wasm/assemblyscript
+subgraph:
+  file: ../subgraph/subgraph.yaml
+  id: MY_ID

packages/client-js/src/__tests__/manifest/migrator/format-0.0.1-alpha.2/web3api.yml

@@ -0,0 +1,12 @@
+description: Awesome Web3API project
+format: "0.0.1-alpha.2"
+api: 
+  schemas:
+    - file: ../query/schema.graphql
+    - file: ../mutation/schema.graphql
+  query:
+    file: ../query/index.ts
+    language: wasm/assemblyscript
+  mutation:
+    file: ../mutation/index.ts
+    language: wasm/assemblyscript

packages/client-js/src/__tests__/manifest/migrator/src/mutation/index.ts

@@ -0,0 +1,5 @@
+import { Ethereum } from '@web3api/wasm-ts'
+
+export function setInformation(): string {
+  return Ethereum.sendTransaction("0x", "myMethod", "");
+}