Merge pull request #33 from cucumber/restore-wasm

Restore WASM support

.github/workflows/test-javascript.yml

@@ -23,6 +23,10 @@ jobs:
       - name: set git core.autocrlf to 'input'
         run: git config --global core.autocrlf input
       - uses: actions/checkout@v3
+      - uses: mymindstorm/setup-emsdk@v11
+        with:
+          # See https://github.com/tree-sitter/tree-sitter/blob/master/cli/emscripten-version
+          version: 2.0.24
       - name: with Node.js ${{ matrix.node-version }} on ${{ matrix.os }}
         uses: actions/setup-node@v3
         with:

CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+### Added
+- Added back wasm support that was deleted in 0.13.0 
+
 ## [0.13.0] - 2022-04-22
 ### Changed
 - Renamed `StepDocument` to `Suggestion`

CONTRIBUTING.md

@@ -2,6 +2,10 @@
 
 Thank you for contributing! Please follow these steps before preparing a pull request
 
+## Prerequisites
+
+You need docker installed. This is because the build process uses docker to build wasm files.
+
 ## Run the tests
 
     npm install
@@ -11,13 +15,16 @@ Thank you for contributing! Please follow these steps before preparing a pull re
 
 If your contribution is to add support for a new programming language, follow these steps:
 
-1. Add the following files, porting the names and expressions from one of the existing implementations:
+1. Run `npm install -E tree-sitter-{language}`
+2. Update `languages` in `scripts/build.js`
+3. Run `npm install` - this should build a wasm for the new language into `dist/{language}.wasm`
+4. Add the following files, porting the names and expressions from one of the existing implementations:
    - `test/tree-sitter/testdata/{language}/ParameterTypes.{ext}`
    - `test/tree-sitter/testdata/{language}/StepDefinitions.{ext}`
-2. Add a new `src/tree-sitter/{language}Language.ts` file
-3. Add the name of the new language to the `LanguageName` type
-4. Update `treeSitterLanguageByName`
-5. Run tests
+5. Add a new `src/tree-sitter/{language}Language.ts` file
+6. Add the name of the new language to the `LanguageName` type
+7. Update `treeSitterLanguageByName` in `src/tree-sitter/ExpressionBuilder.ts`
+8. Run tests
 
 As you are working on step 1 and 2 - use [tree-sitter playground](https://tree-sitter.github.io/tree-sitter/playground)
 to build your query. The queries must have [capturing nodes](https://tree-sitter.github.io/tree-sitter/using-parsers#query-syntax):

package.json

@@ -8,14 +8,22 @@
   "files": [
     "dist/cjs",
     "dist/esm",
-    "*.wasm"
+    "dist/*.wasm"
   ],
   "module": "dist/esm/src/index.js",
   "jsnext:main": "dist/esm/src/index.js",
   "exports": {
     ".": {
       "import": "./dist/esm/src/index.js",
       "require": "./dist/cjs/src/index.js"
+    },
+    "./node": {
+      "import": "./dist/esm/src/tree-sitter-node/NodeParserAdapter.js",
+      "require": "./dist/cjs/src/tree-sitter-node/NodeParserAdapter.js"
+    },
+    "./wasm": {
+      "import": "./dist/esm/src/tree-sitter-wasm/WasmParserAdapter.js",
+      "require": "./dist/cjs/src/tree-sitter-wasm/WasmParserAdapter.js"
     }
   },
   "scripts": {
@@ -25,11 +33,12 @@
     "test": "mocha && npm run test:cjs",
     "test:cjs": "npm run build:cjs && mocha --config .mocharc.cjs.json --recursive dist/cjs/test",
     "prepublishOnly": "npm run build",
-    "eslint-fix": "eslint --ext ts,tsx --max-warnings 0 --fix src test",
-    "eslint": "eslint --ext ts,tsx --max-warnings 0 src test",
+    "eslint-fix": "eslint --ext ts --max-warnings 0 --fix src test",
+    "eslint": "eslint --ext ts --max-warnings 0 src test",
     "upgrade": "npm-check-updates --upgrade",
     "prepare": "husky install",
-    "pretty-quick-staged": "pretty-quick --staged"
+    "pretty-quick-staged": "pretty-quick --staged",
+    "postinstall": "scripts/build.js && cp node_modules/web-tree-sitter/tree-sitter.wasm dist"
   },
   "repository": {
     "type": "git",
@@ -66,9 +75,14 @@
     "npm-check-updates": "12.5.9",
     "prettier": "2.6.2",
     "pretty-quick": "3.1.3",
+    "tree-sitter-cli": "0.20.6",
     "ts-node": "10.7.0",
     "txtgen": "3.0.1",
-    "typescript": "4.6.3"
+    "typescript": "4.6.3",
+    "web-tree-sitter": "0.20.5"
+  },
+  "peerDependencies": {
+    "web-tree-sitter": "^0.20.5"
   },
   "dependencies": {
     "@cucumber/cucumber-expressions": "^15.1.1",

scripts/build.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+
+import { exec } from 'child_process'
+import fs from 'fs'
+import path from 'path'
+
+const languages = {
+  typescript: ['typescript', 'typescript'],
+  java: ['java'],
+}
+
+// Build wasm parsers for supported languages
+const parsersDir = 'dist'
+if (!fs.existsSync(parsersDir)) {
+  fs.mkdirSync(parsersDir)
+}
+for (const [lang, names] of Object.entries(languages)) {
+  const [moduleName, ...variant] = names
+  const module = path.join('node_modules/tree-sitter-' + moduleName, ...variant)
+  const output = 'tree-sitter-' + names[names.length - 1] + '.wasm'
+
+  let command
+  if (process.env.CI) {
+    console.log(`Compiling ${lang} parser`)
+    command = `node_modules/.bin/tree-sitter build-wasm ${module}`
+  } else {
+    console.log(`Compiling ${lang} parser inside docker`)
+    // https://github.com/tree-sitter/tree-sitter/issues/1560
+    command = `node_modules/.bin/tree-sitter build-wasm ${module} --docker`
+  }
+  exec(command, (err) => {
+    if (err) {
+      console.error('Failed to build wasm for ' + lang + ': ' + err.message)
+      process.exit(1)
+    } else {
+      const newPath = `${parsersDir}/${lang}.wasm`
+      fs.rename(output, newPath, (err) => {
+        if (err) {
+          console.error('Failed to copy built parser: ' + err.message)
+          process.exit(1)
+        } else console.log(`Successfully compiled ${newPath}`)
+      })
+    }
+  })
+}

src/tree-sitter/NodeParserAdapter.ts → src/tree-sitter-node/NodeParserAdapter.ts

@@ -4,9 +4,9 @@ import Java from 'tree-sitter-java'
 // @ts-ignore
 import TypeScript from 'tree-sitter-typescript'
 
-import { LanguageName, ParserAdpater } from './types'
+import { LanguageName, ParserAdapter } from '../tree-sitter/types'
 
-export class NodeParserAdapter implements ParserAdpater {
+export class NodeParserAdapter implements ParserAdapter {
   readonly parser = new Parser()
 
   query(source: string): Parser.Query {

src/tree-sitter-wasm/WasmParserAdapter.ts

@@ -0,0 +1,34 @@
+import NodeParser from 'tree-sitter'
+import Parser from 'web-tree-sitter'
+
+import { LanguageName, LanguageNames, ParserAdapter } from '../tree-sitter/types.js'
+
+export class WasmParserAdapter implements ParserAdapter {
+  public parser: NodeParser
+  private languages: Record<LanguageName, Parser.Language>
+
+  async init(wasmBaseUrl: string) {
+    await Parser.init()
+    // @ts-ignore
+    this.parser = new Parser()
+
+    const languages = await Promise.all(
+      LanguageNames.map((languageName) => {
+        const wasmUrl = `${wasmBaseUrl}/${languageName}.wasm`
+        return Parser.Language.load(wasmUrl)
+      })
+    )
+    // @ts-ignore
+    this.languages = Object.fromEntries(
+      LanguageNames.map((languageName, i) => [languageName as LanguageName, languages[i]])
+    )
+  }
+
+  query(source: string): NodeParser.Query {
+    return this.parser.getLanguage().query(source)
+  }
+
+  setLanguage(language: LanguageName): void {
+    this.parser.setLanguage(this.languages[language])
+  }
+}

src/tree-sitter/ExpressionBuilder.ts

@@ -10,7 +10,7 @@ import { javaLanguage } from './javaLanguage.js'
 import {
   LanguageName,
   ParameterTypeMeta,
-  ParserAdpater,
+  ParserAdapter,
   Source,
   TreeSitterLanguage,
 } from './types.js'
@@ -25,7 +25,7 @@ const defineStepDefinitionQueryKeys = <const>['expression']
 const defineParameterTypeKeys = <const>['name', 'expression']
 
 export class ExpressionBuilder {
-  constructor(private readonly parserAdpater: ParserAdpater) {}
+  constructor(private readonly parserAdpater: ParserAdapter) {}
 
   build(
     sources: readonly Source[],

src/tree-sitter/index.ts

@@ -1,3 +1,2 @@
 export * from './ExpressionBuilder.js'
-export * from './NodeParserAdapter.js'
 export * from './types.js'

src/tree-sitter/types.ts

@@ -2,7 +2,8 @@ import Parser from 'tree-sitter'
 
 export type ParameterTypeMeta = { name: string; regexp: string }
 
-export type LanguageName = 'java' | 'typescript'
+export const LanguageNames = ['java', 'typescript'] as const
+export type LanguageName = typeof LanguageNames[number]
 
 export type Source = {
   language: LanguageName
@@ -19,7 +20,7 @@ export type TreeSitterLanguage = {
  * The Node.js and Web bindings have slightly different APIs. We hide this difference behind this interface.
  * https://github.com/tree-sitter/node-tree-sitter/issues/68
  */
-export interface ParserAdpater {
+export interface ParserAdapter {
   readonly parser: Parser
   setLanguage(language: LanguageName): void
   query(source: string): Parser.Query

test/tree-sitter/ExpressionBuilder.test.ts

@@ -5,10 +5,16 @@ import glob from 'glob'
 import path from 'path'
 
 import { ExpressionBuilder, LanguageName } from '../../src/index.js'
-import { NodeParserAdapter } from '../../src/tree-sitter/NodeParserAdapter.js'
+import { ParserAdapter } from '../../src/tree-sitter/types'
+import { NodeParserAdapter } from '../../src/tree-sitter-node/NodeParserAdapter.js'
+import { WasmParserAdapter } from '../../src/tree-sitter-wasm/WasmParserAdapter.js'
 
-describe('ExpressionBuilder', () => {
-  const expressionBuilder = new ExpressionBuilder(new NodeParserAdapter())
+function defineContract(makeParserAdapter: () => Promise<ParserAdapter>) {
+  let expressionBuilder: ExpressionBuilder
+  beforeEach(async () => {
+    const parserAdpater = await makeParserAdapter()
+    expressionBuilder = new ExpressionBuilder(parserAdpater)
+  })
 
   for (const dir of glob.sync(`test/tree-sitter/testdata/*`)) {
     const language = path.basename(dir) as LanguageName
@@ -27,4 +33,18 @@ describe('ExpressionBuilder', () => {
       )
     })
   }
+}
+
+describe('ExpressionBuilder', () => {
+  context('with NodeParserAdapter', () => {
+    defineContract(() => Promise.resolve(new NodeParserAdapter()))
+  })
+
+  context('with WasmParserAdapter', () => {
+    defineContract(async () => {
+      const wasmParserAdapter = new WasmParserAdapter()
+      await wasmParserAdapter.init('dist')
+      return wasmParserAdapter
+    })
+  })
 })