Introduce project manifest version 0.0.2 (#495)

* Introduce project manifest version 0.0.2 - Define new project manifest schema - Update validator for new schema, validate chain types - Update CLI for new schema - Update `init` to create a v0.0.2 project - Update validate to validate all supported versions - Update Node for new schema - Rely on genesis hash for network identification - Add `--network-registry` flag to provide mapping from genesis hash to endpoint - Support new chain types in external file - Support new entry point for indexer sandbox * Fix up tests * Fix up prettier * Fix lint issue * Enable noImplicitAny ts setting for cli, common and validator * Update new manifest version from 0.0.2 to 0.2.0, address other minor comments * Add tests for index manager, reuse sandbox VM for same entry point * Revert CLI init changes * Update docs * Update init command to allow specifying spec version * Remove networkDictionary option and require networkEndpoint to be provided for v0.2.0 projects * Remove logging * Add validator tests to jest scrope, fix valitation when project.yaml is missing * Fix failing test * Allow specifying an optional endpoint in network config, update CLI to resolve genesis hash from url * Fix commit hook issue with prettier and partially staged files * Fix tests failing * Disable breaking tests * Remove name from datasource in v0.2.0 manifest spec
subquery · Oct 4, 2021 · 20826f8 · 20826f8
1 parent 846f8e5
commit 20826f8
Show file tree

Hide file tree

Showing 60 changed files with 1,034 additions and 407 deletions.
diff --git a/.husky/pre-commit b/.husky/pre-commit
@@ -1,5 +1,4 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
 
-# concurrent disabled to eslint and prettier work together
-npx lint-staged --concurrent false
+npx lint-staged
diff --git a/docs/references/references.md b/docs/references/references.md
@@ -35,18 +35,20 @@ Options:
       --timestamp-field     Enable/disable created_at and updated_at in schema
                                                        [boolean] [default: true]
   -d, --network-dictionary  Specify the dictionary api for this network [string]
-  ```
+      --proof-of-index      Enable/disable proof of index
+                                                      [boolean] [default: false]
+```
 
 ### --version
 
 This displays the current version.
 
-``` shell
+```shell
 > subql-node --version
 0.19.1
 ```
 
-### -f, --subquery 
+### -f, --subquery
 
 Use this flag to start the SubQuery project.
 
@@ -64,7 +66,8 @@ subql-node -f . --subquery-name=test2
 ```
 
 ### -c, --config
-All these various configurations can be placed into a .yml or .json file and then referenced with the config flag. 
+
+All these various configurations can be placed into a .yml or .json file and then referenced with the config flag.
 
 Sample subquery_config.yml file:
 
@@ -79,6 +82,7 @@ Place this file in the same directory as the project. Then in the current projec
 ```shell
 > subql-node -c ./subquery_config.yml
 ```
+
 ### --local
 
 This flag is primarily used for debugging purposes where it creates the default starter_entity table in the default "postgres" schema.
@@ -91,14 +95,14 @@ Note that once you use this flag, removing it won't mean that it will point to a
 
 ### --batch-size
 
-This flag allows you to set the batch size in the command line. If batch size is also set in the config file, this takes precedent. 
+This flag allows you to set the batch size in the command line. If batch size is also set in the config file, this takes precedent.
 
 ```shell
 > subql-node -f . --batch-size=20
-2021-08-09T23:24:43.775Z <fetch> INFO fetch block [6601,6620], total 20 blocks 
-2021-08-09T23:24:45.606Z <fetch> INFO fetch block [6621,6640], total 20 blocks 
-2021-08-09T23:24:47.415Z <fetch> INFO fetch block [6641,6660], total 20 blocks 
-2021-08-09T23:24:49.235Z <fetch> INFO fetch block [6661,6680], total 20 blocks 
+2021-08-09T23:24:43.775Z <fetch> INFO fetch block [6601,6620], total 20 blocks
+2021-08-09T23:24:45.606Z <fetch> INFO fetch block [6621,6640], total 20 blocks
+2021-08-09T23:24:47.415Z <fetch> INFO fetch block [6641,6660], total 20 blocks
+2021-08-09T23:24:49.235Z <fetch> INFO fetch block [6661,6680], total 20 blocks
 ```
 
 <!-- ### --timeout -->
@@ -109,9 +113,9 @@ This outputs debug information to the console output and forcefully sets the log
 
 ```shell
 > subql-node -f . --debug
-2021-08-10T11:45:39.471Z <db> DEBUG Executing (1b0d0c23-d7c7-4adb-a703-e4e5c414e035): INSERT INTO "subquery_1"."starter_entities" ("id","block_height","created_at","updated_at") VALUES ($1,$2,$3,$4) ON CONFLICT ("id") DO UPDATE SET "id"=EXCLUDED."id","block_height"=EXCLUDED."block_height","updated_at"=EXCLUDED."updated_at" RETURNING "id","block_height","created_at","updated_at"; 
-2021-08-10T11:45:39.472Z <db> DEBUG Executing (default): UPDATE "subqueries" SET "next_block_height"=$1,"updated_at"=$2 WHERE "id" = $3 
-2021-08-10T11:45:39.472Z <db> DEBUG Executing (1b0d0c23-d7c7-4adb-a703-e4e5c414e035): COMMIT; 
+2021-08-10T11:45:39.471Z <db> DEBUG Executing (1b0d0c23-d7c7-4adb-a703-e4e5c414e035): INSERT INTO "subquery_1"."starter_entities" ("id","block_height","created_at","updated_at") VALUES ($1,$2,$3,$4) ON CONFLICT ("id") DO UPDATE SET "id"=EXCLUDED."id","block_height"=EXCLUDED."block_height","updated_at"=EXCLUDED."updated_at" RETURNING "id","block_height","created_at","updated_at";
+2021-08-10T11:45:39.472Z <db> DEBUG Executing (default): UPDATE "subqueries" SET "next_block_height"=$1,"updated_at"=$2 WHERE "id" = $3
+2021-08-10T11:45:39.472Z <db> DEBUG Executing (1b0d0c23-d7c7-4adb-a703-e4e5c414e035): COMMIT;
 ```
 
 ### --profiler
@@ -120,15 +124,15 @@ This shows profiler information.
 
 ```shell
 subql-node -f . --local --profiler
-2021-08-10T10:57:07.234Z <profiler> INFO FetchService, fetchMeta, 3876 ms 
-2021-08-10T10:57:08.095Z <profiler> INFO FetchService, fetchMeta, 774 ms 
-2021-08-10T10:57:10.361Z <profiler> INFO SubstrateUtil, fetchBlocksBatches, 2265 ms 
-2021-08-10T10:57:10.361Z <fetch> INFO fetch block [3801,3900], total 100 blocks 
+2021-08-10T10:57:07.234Z <profiler> INFO FetchService, fetchMeta, 3876 ms
+2021-08-10T10:57:08.095Z <profiler> INFO FetchService, fetchMeta, 774 ms
+2021-08-10T10:57:10.361Z <profiler> INFO SubstrateUtil, fetchBlocksBatches, 2265 ms
+2021-08-10T10:57:10.361Z <fetch> INFO fetch block [3801,3900], total 100 blocks
 ```
 
 ### --network-endpoint
 
-This flag allows users to override the network endpoint configuration from the manifest file. 
+This flag allows users to override the network endpoint configuration from the manifest file.
 
 ```shell
 subql-node -f . --network-endpoint="wss://polkadot.api.onfinality.io/public-ws"
@@ -139,9 +143,9 @@ Note that this must also be set in the manifest file, otherwise you'll get:
 ```shell
 ERROR Create Subquery project from given path failed! Error: failed to parse project.yaml.
 An instance of ProjectManifestImpl has failed the validation:
- - property network has failed the following constraints: isObject 
- - property network.network has failed the following constraints: nestedValidation 
- ```
+ - property network has failed the following constraints: isObject
+ - property network.network has failed the following constraints: nestedValidation
+```
 
 ### --output-fmt
 
@@ -154,14 +158,15 @@ There are two different terminal output formats. JSON or colored. Colored is the
 
 ```shell
 > subql-node -f . --output-fmt=colored
-2021-08-10T11:57:41.480Z <subql-node> INFO node started 
+2021-08-10T11:57:41.480Z <subql-node> INFO node started
 (node:24707) [PINODEP007] Warning: bindings.level is deprecated, use options.level option instead
-2021-08-10T11:57:48.981Z <fetch> INFO fetch block [10201,10300], total 100 blocks 
-2021-08-10T11:57:51.862Z <fetch> INFO fetch block [10301,10400], total 100 blocks 
+2021-08-10T11:57:48.981Z <fetch> INFO fetch block [10201,10300], total 100 blocks
+2021-08-10T11:57:51.862Z <fetch> INFO fetch block [10301,10400], total 100 blocks
 ```
+
 ### --log-level
 
-There are 7 options to choose from. “fatal”, “error”, “warn”, “info”, “debug”, “trace”, “silent”. The example below shows silent. Nothing will be printed in the terminal so the only way to tell if the node is working or not is to query the database for row count (select count(*) from subquery_1.starter_entities) or query the block height. 
+There are 7 options to choose from. “fatal”, “error”, “warn”, “info”, “debug”, “trace”, “silent”. The example below shows silent. Nothing will be printed in the terminal so the only way to tell if the node is working or not is to query the database for row count (select count(\*) from subquery_1.starter_entities) or query the block height.
 
 ```shell
 > subql-node -f . --log-level=silent
@@ -181,7 +186,6 @@ There are 7 options to choose from. “fatal”, “error”, “warn”, “inf
 
 <!-- ### --migrate TBA -->
 
-
 ### --timestamp-field
 
 By default this is true. when set to false with:
@@ -204,10 +208,10 @@ subql-node -f . -d "https://api.subquery.network/sq/subquery/dictionary-polkadot
 
 [Read more about how a SubQuery Dictionary works](../tutorials_examples/dictionary.md).
 
-
 ## subql-query
 
 ### --help
+
 This shows the help options.
 
 ```shell
@@ -228,27 +232,28 @@ ns:
 
 This displays the current version.
 
-``` shell
+```shell
 > subql-query --version
 0.7.0
 ```
+
 ### -n, --name
 
-This flag is used to start the query service. If the --subquery-name flag is not provided when running an indexer, the name here will refer to the default project name. If --subquery-name is set, then the name here should match what was set. 
+This flag is used to start the query service. If the --subquery-name flag is not provided when running an indexer, the name here will refer to the default project name. If --subquery-name is set, then the name here should match what was set.
 
-``` shell
+```shell
 > subql-node -f . // --subquery-name not set
 
 > subql-query -n subql-helloworld  --playground // the name defaults to the project directory name
 ```
 
-``` shell
+```shell
 > subql-node -f . --subquery-name=hiworld // --subquery-name set
 
 > subql-query -n hiworld --playground  // the name points to the subql-helloworld project but with the name of hiworld
 ```
 
-### --playground 
+### --playground
 
 This flag enables the graphql playground so should always be included by default to be of any use.
 
@@ -260,4 +265,4 @@ See [--output-fmt](https://doc.subquery.network/references/references.html#outpu
 
 See [--log-level](https://doc.subquery.network/references/references.html#log-level)
 
-<!-- ### --indexer TBA -->
+<!-- ### --indexer TBA -->
diff --git a/jest.config.js b/jest.config.js
@@ -20,22 +20,23 @@ module.exports = {
   // collectCoverage: false,
 
   // An array of glob patterns indicating a set of files for which coverage information should be collected
-  "collectCoverageFrom": [
-    "packages/cli/src/**/*.ts",
-    "packages/common/src/**/*.ts",
-    "packages/node/src/**/*.ts",
+  collectCoverageFrom: [
+    'packages/cli/src/**/*.ts',
+    'packages/common/src/**/*.ts',
+    'packages/node/src/**/*.ts',
+    'packages/validator/src/**/*.ts',
   ],
 
   // The directory where Jest should output its coverage files
-  coverageDirectory: "coverage",
+  coverageDirectory: 'coverage',
 
   // An array of regexp pattern strings used to skip coverage collection
   // coveragePathIgnorePatterns: [
   //   "/node_modules/"
   // ],
 
   // Indicates which provider should be used to instrument code for coverage
-  coverageProvider: "v8",
+  coverageProvider: 'v8',
 
   // A list of reporter names that Jest uses when writing coverage reports
   // coverageReporters: [
@@ -81,17 +82,13 @@ module.exports = {
 
   // An array of file extensions your modules use
 
-  "moduleFileExtensions": [
-    "js",
-    "json",
-    "ts"
-  ],
+  moduleFileExtensions: ['js', 'json', 'ts'],
 
   // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
   // moduleNameMapper: {},
   moduleNameMapper: {
-    "^@subql/common": '<rootDir>/packages/common/src',
-    "^@subql/common/(.*)$": '<rootDir>/packages/common/src/$1',
+    '^@subql/common': '<rootDir>/packages/common/src',
+    '^@subql/common/(.*)$': '<rootDir>/packages/common/src/$1',
   },
 
   // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
@@ -138,9 +135,7 @@ module.exports = {
 
   // The paths to modules that run some code to configure or set up the testing environment before each test
   // setupFiles: [],
-  "setupFiles": [
-    "./test/jest-setup.ts"
-  ],
+  setupFiles: ['./test/jest-setup.ts'],
 
   // A list of paths to modules that run some code to configure or set up the testing framework before each test
   // setupFilesAfterEnv: [],
@@ -152,7 +147,7 @@ module.exports = {
   // snapshotSerializers: [],
 
   // The test environment that will be used for testing
-  testEnvironment: "node",
+  testEnvironment: 'node',
 
   // Options that will be passed to the testEnvironment
   // testEnvironmentOptions: {},
@@ -172,7 +167,7 @@ module.exports = {
   // ],
 
   // The regexp pattern or array of patterns that Jest uses to detect test files
-  "testRegex": ".*\\.spec\\.ts$",
+  testRegex: '.*\\.spec\\.ts$',
 
   // This option allows the use of a custom results processor
   // testResultsProcessor: undefined,
@@ -189,7 +184,7 @@ module.exports = {
   // A map from regular expressions to paths to transformers
   transform: {
     '^.+\\.(ts|tsx)?$': 'ts-jest',
-    "^.+\\.(js|jsx)$": "babel-jest",
+    '^.+\\.(js|jsx)$': 'babel-jest',
   },
 
   // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
@@ -198,9 +193,7 @@ module.exports = {
   //   "\\.pnp\\.[^\\/]+$"
   // ],
 
-  "transformIgnorePatterns": [
-    "node_modules/(?!(@polkadot|@babel/runtime/helpers/esm)/)"
-  ],
+  transformIgnorePatterns: ['node_modules/(?!(@polkadot|@babel/runtime/helpers/esm)/)'],
 
   // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
   // unmockedModulePathPatterns: undefined,

diff --git a/package.json b/package.json
@@ -48,7 +48,12 @@
     "vuepress-theme-hope": "^1.20.2"
   },
   "lint-staged": {
-    "*.ts": "eslint --cache --fix",
-    "*": "pretty-quick --staged --pattern 'packages/*/src/**/*"
+    "*.ts": [
+      "eslint --cache --fix",
+      "prettier --write"
+    ],
+    "packages/**/*.!(*ts)": [
+      "prettier --write"
+    ]
   }
 }
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -11,6 +11,7 @@
     "@oclif/command": "^1.8.0",
     "@oclif/config": "^1.17.0",
     "@oclif/plugin-help": "^3.2.2",
+    "@polkadot/api": "^5.9.1",
     "@subql/common": "workspace:*",
     "@subql/validator": "workspace:*",
     "@types/ejs": "^3.0.6",

diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts
@@ -5,8 +5,8 @@ import fs from 'fs';
 import path from 'path';
 import {Command, flags} from '@oclif/command';
 import cli from 'cli-ux';
-import {createProject, installDependencies} from '../controller/init-controller';
-import {ProjectSpec} from '../types';
+import {createProject, installDependencies, getGenesisHash} from '../controller/init-controller';
+import {ProjectSpecBase, ProjectSpecV0_2_0} from '../types';
 
 export default class Init extends Command {
   static description = 'Init a scaffold subquery project';
@@ -19,6 +19,11 @@ export default class Init extends Command {
     location: flags.string({char: 'l', description: 'local folder to create the project in'}),
     'install-dependencies': flags.boolean({description: 'Install dependencies as well', default: false}),
     npm: flags.boolean({description: 'Force using NPM instead of yarn, only works with `install-dependencies` flag'}),
+    specVersion: flags.string({
+      required: false,
+      default: '0.0.1',
+      description: 'The spec version to be used by the project',
+    }),
   };
 
   static args = [
@@ -30,7 +35,7 @@ export default class Init extends Command {
 
   async run(): Promise<void> {
     const {args, flags} = this.parse(Init);
-    const project = {} as ProjectSpec;
+    const project = {} as ProjectSpecBase;
 
     const location = flags.location ? path.resolve(flags.location) : process.cwd();
 
@@ -41,10 +46,16 @@ export default class Init extends Command {
       throw new Error(`Directory ${project.name} exists, try another project name`);
     }
     project.repository = await cli.prompt('Git repository', {required: false});
+
     project.endpoint = await cli.prompt('RPC endpoint', {
       default: 'wss://polkadot.api.onfinality.io/public-ws',
       required: true,
     });
+
+    if (flags.specVersion === '0.2.0') {
+      (project as ProjectSpecV0_2_0).genesisHash = await getGenesisHash(project.endpoint);
+    }
+
     project.author = await cli.prompt('Authors', {required: true});
     project.description = await cli.prompt('Description', {required: false});
     project.version = await cli.prompt('Version:', {default: '1.0.0', required: true});

diff --git a/packages/cli/src/controller/codegen-controller.ts b/packages/cli/src/controller/codegen-controller.ts
@@ -39,7 +39,7 @@ export interface ProcessedField {
   isJsonInterface: boolean;
 }
 
-export async function generateJsonInterfaces(projectPath, schema: string) {
+export async function generateJsonInterfaces(projectPath: string, schema: string): Promise<void> {
   const typesDir = path.join(projectPath, TYPE_ROOT_DIR);
   const jsonObjects = getAllJsonObjects(schema);
   const jsonInterfaces = jsonObjects.map((r) => {