Merge pull request #2717 from terascope/node-12-14

v0.77.0 - Update to node support to 12/14

.travis.yml

@@ -2,7 +2,7 @@
 dist: bionic
 os: linux
 language: node_js
-node_js: '12.20'
+node_js: '14.17'
 # use a smaller depth for faster builds
 git:
   depth: 10
@@ -23,10 +23,10 @@ jobs:
   fast_finish: true
 
   include:
-    - stage: "Verify"
+    - stage: 'Verify'
       name: 'Verify Build (node 10)'
       # use node 10 to make sure we don't lose compatibility
-      node_js: '10.19'
+      node_js: '12.20'
       # run on any pull_request
       if: branch = master AND type IN (pull_request)
       # only cache on verify build
@@ -39,10 +39,10 @@ jobs:
         - yarn lint
         - yarn sync --verify
 
-    - stage: "Tests"
+    - stage: 'Tests'
       name: 'Unit Test Suite (node 10)'
       # use node 10 to make sure we don't lose compatibility
-      node_js: '10.19'
+      node_js: '12.20'
       # run only on pull-requests or cron
       if: branch = master AND type IN (pull_request, cron)
       script: yarn --silent test --suite unit-a
@@ -53,40 +53,42 @@ jobs:
       if: branch = master AND type IN (pull_request, cron)
       script: yarn --silent test --suite unit-b
 
-    - script:
-      name: 'ES Test Suite (elasticsearch 6) (node 12)'
+    - name: 'ES Test Suite (elasticsearch 6) (node 12)'
       node_js: '12.20'
       # run only on pull-requests and cron
       if: branch = master AND type IN (pull_request, cron)
       script: yarn --silent test --suite elasticsearch --elasticsearch-version 6.8.6 --elasticsearch-api-version 6.5
 
-    - script:
-      name: 'ES Test Suite (elasticsearch 7) (node 12)'
-      node_js: '12.20'
+    - name: 'Unit Test Suite (node 14)'
+      node_js: '14.17'
+      # run only on pull-requests or cron
+      if: branch = master AND type IN (pull_request, cron)
+      script: yarn --silent test --suite unit-b
+
+    - name: 'ES Test Suite (elasticsearch 7) (node 14)'
+      node_js: '14.17'
       # run only on pull-requests
       if: branch = master AND type IN (pull_request) AND commit_message !~ /^WIP/
       script: yarn --silent test --suite elasticsearch --elasticsearch-version 7.2.1 --elasticsearch-api-version 7.0 --report-coverage false
 
-    - script:
-      name: 'End-to-End Test Suite (elasticsearch 6) (node 12)'
-      node_js: '12.20'
+    - name: 'End-to-End Test Suite (elasticsearch 6) (node 14)'
+      node_js: '14.17'
       # run only on pull-requests and cron
       if: branch = master AND type IN (pull_request, cron) AND fork = false
       script:
         - export SERVICE_HEAP_OPTS="-Xms768m -Xmx768m"
         - yarn --silent --cwd e2e test
 
-    - stage: "Releases"
+    - stage: 'Releases'
       name: 'Publish prerelease packages and docker image'
       # Run on pull-request and when the commit message includes a prerelease bump
       if: branch = master AND type IN (pull_request) AND commit_message =~ /(bump|release):.*\((premajor|preminor|prepatch|prerelease)\).*/
       script:
         - yarn ts-scripts publish -t prerelease npm
         - yarn ts-scripts publish -t prerelease docker
 
-    - script:
-      name: 'Publish packages, docs and expiremental docker image'
-      node_js: '12.20'
+    - name: 'Publish packages, docs and experimental docker image'
+      node_js: '14.17'
       # run a push to master
       if: tag IS blank AND branch = master AND type NOT IN (pull_request, cron)
       script:
@@ -99,8 +101,7 @@ jobs:
         # do this last
         - ./scripts/publish-documentation.sh
 
-    - script:
-      name: 'Create Tag Release'
+    - name: 'Create Tag Release'
       # run on tagged releases
       if: tag IS present AND type NOT IN (pull_request, cron)
       # no need to run test on tagged release
@@ -120,8 +121,7 @@ jobs:
             tags: true
 
     # Build nightly releases
-    - script:
-      name: 'Daily Docker Builds'
+    - name: 'Daily Docker Builds'
       # run on cron jobs
       if: type = cron
       script: true

Dockerfile

@@ -1,4 +1,4 @@
-FROM terascope/node-base:12.22.1
+FROM terascope/node-base:14.17.0
 
 ENV NODE_ENV production
 

e2e/package.json

@@ -41,6 +41,10 @@
         "signale": "^1.4.0",
         "uuid": "^8.3.2"
     },
+    "engines": {
+        "node": "^12.20.0 || >=14.17.0",
+        "yarn": ">=1.16.0"
+    },
     "terascope": {
         "testSuite": "e2e"
     }

e2e/test/download-assets.js

@@ -6,7 +6,34 @@ const semver = require('semver');
 const downloadRelease = require('@terascope/fetch-github-release');
 const signale = require('./signale');
 
-const nodeVersion = '12';
+/**
+ * This will get the correct teraslice node version so
+ * we can download the correct asset
+*/
+function getNodeVersion() {
+    const dockerFilePath = path.join(__dirname, '..', '..', 'Dockerfile');
+    const dockerFileContents = fs.readFileSync(dockerFilePath, 'utf8');
+    const fromLine = dockerFileContents.split('\n').find((line) => line.trim().startsWith('FROM '));
+    if (!fromLine) {
+        throw new Error('Unable to find the import FROM line in the Dockerfile');
+    }
+
+    const splitChars = 'FROM terascope/node-base:';
+    if (!fromLine.includes(splitChars)) {
+        throw new Error(`Dockerfile does not contain "${splitChars}". If this has changed, this file needs to be changed`);
+    }
+
+    const nodeVersion = fromLine.trim().split(splitChars)[1];
+    const majorNodeVersion = parseInt(nodeVersion.split('.')[0], 10);
+    if (Number.isNaN(majorNodeVersion)) {
+        throw new Error(`Expected to find a valid node major version in the Dockerfile but found ${majorNodeVersion}`);
+    }
+
+    return String(majorNodeVersion);
+}
+
+const nodeVersion = getNodeVersion();
+
 const autoloadDir = path.join(__dirname, '..', 'autoload');
 const leaveZipped = true;
 const disableLogging = true;
@@ -32,6 +59,7 @@ function assetFileInfo(assetName) {
         name,
         version: semver.coerce(version),
         repo: `${name}-assets`,
+        bundle: assetName.includes('-bundle'),
         fileName: assetName
     };
 }
@@ -41,7 +69,7 @@ function getOlderAsset(assets, assetName) {
 
     return assets.find((a) => {
         if (a.name !== name) return false;
-        return semver.gt(version, a.version);
+        return semver.gt(version, a.version) && a.bundle;
     });
 }
 
@@ -50,8 +78,12 @@ function filterRelease(release) {
 }
 
 function filterAsset(asset) {
+    // if it includes the bundle choose that
+    if (asset.name.includes(`node-${nodeVersion}-bundle.zip`)) {
+        return true;
+    }
     const mustContain = `node-${nodeVersion}-linux-x64.zip`;
-    return asset.name.indexOf(mustContain) >= 0;
+    return asset.name.includes(mustContain);
 }
 
 function listAssets() {
@@ -78,41 +110,53 @@ function deleteOlderAssets() {
         return c > 1;
     });
 
-    const olderAssets = duplicateAssets.reduce((acc, current, index, src) => {
-        const without = src.filter((a, i) => index !== i);
-        const older = getOlderAsset(without, current.fileName);
-        if (older) {
-            older.newerVersion = current.version;
-            return acc;
-        }
-        return acc.concat([current]);
-    }, []);
+    const olderAssets = duplicateAssets
+        .reduce((acc, current, index, src) => {
+            const without = src.filter((a, i) => index !== i);
+            const older = getOlderAsset(without, current.fileName);
+            if (older) {
+                older.newerVersion = current.version;
+                return acc;
+            }
+            return acc.concat([current]);
+        }, [])
+        .filter(({ name, newerVersion, bundle }, index, arr) => {
+            if (newerVersion == null && bundle) {
+                return arr.find((a, i) => i !== index && a.name === name && a.bundle);
+            }
+            return newerVersion == null;
+        });
 
     for (const asset of olderAssets) {
-        signale.warn(`Deleting asset ${asset.name}@v${asset.version} in-favor of existing v${asset.newerVersion}`);
+        const b = asset.bundle ? ' [bundle]' : ' [non-bundle]';
+        signale.warn(`Deleting asset ${asset.name}@v${asset.version} in-favor of existing v${asset.newerVersion || asset.version}${b}`);
         fs.unlinkSync(path.join(autoloadDir, asset.fileName));
     }
 }
 
 function logAssets() {
-    const assets = listAssets().map(({ name, version }) => `${name}@v${version}`);
+    const assets = listAssets().map(({ name, bundle, version }) => {
+        if (bundle) return `${name}@v${version} [bundle]`;
+        return `${name}@v${version} [non-bundle]`;
+    });
+    if (!assets.length) return;
+
     signale.info(`Autoload asset bundles: ${assets.join(', ')}`);
 }
 
+/**
+ * @todo change this to not download both the bundled and non-bundled versions
+*/
 async function downloadAssets() {
-    const promises = bundles.map(async ({ repo }) => {
-        await downloadRelease(
-            'terascope',
-            repo,
-            autoloadDir,
-            filterRelease,
-            filterAsset,
-            leaveZipped,
-            disableLogging
-        );
-    });
-
-    await Promise.all(promises);
+    await Promise.all(bundles.map(({ repo }) => downloadRelease(
+        'terascope',
+        repo,
+        autoloadDir,
+        filterRelease,
+        filterAsset,
+        leaveZipped,
+        disableLogging
+    )));
 
     deleteOlderAssets();
     logAssets();

jsconfig.json

@@ -1,6 +1,6 @@
 {
     "compilerOptions": {
-        "target": "es2018",
+        "target": "es2019",
         "module": "commonjs",
         "baseUrl": "./packages"
     }

package.json

@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "0.76.1",
+    "version": "0.77.0",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {
@@ -55,7 +55,7 @@
         "typescript": "~4.2.4"
     },
     "engines": {
-        "node": ">=10.16.0",
+        "node": "^12.20.0 || >=14.17.0",
         "yarn": ">=1.16.0"
     },
     "os": [
@@ -66,7 +66,7 @@
     "terascope": {
         "root": true,
         "type": "monorepo",
-        "target": "es2018",
+        "target": "es2019",
         "tests": {
             "suites": {
                 "e2e": [
@@ -90,6 +90,10 @@
         },
         "npm": {
             "registry": "https://registry.npmjs.org/"
+        },
+        "engines": {
+            "node": "^12.20.0 || >=14.17.0",
+            "yarn": ">=1.16.0"
         }
     }
 }

packages/data-mate/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-mate",
     "displayName": "Data-Mate",
-    "version": "0.27.4",
+    "version": "0.28.0",
     "description": "Library of data validations/transformations",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme",
     "repository": {
@@ -29,9 +29,9 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/data-types": "^0.28.1",
-        "@terascope/types": "^0.8.0",
-        "@terascope/utils": "^0.38.1",
+        "@terascope/data-types": "^0.29.0",
+        "@terascope/types": "^0.9.0",
+        "@terascope/utils": "^0.39.0",
         "@turf/bbox": "^6.2.0",
         "@turf/bbox-polygon": "^6.3.0",
         "@turf/boolean-point-in-polygon": "^6.2.0",
@@ -50,15 +50,16 @@
         "uuid": "^8.3.2",
         "valid-url": "^1.0.9",
         "validator": "^13.6.0",
-        "xlucene-parser": "^0.35.1"
+        "xlucene-parser": "^0.36.0"
     },
     "devDependencies": {
         "@types/ip6addr": "^0.2.2",
         "@types/uuid": "^8.3.0",
         "benchmark": "^2.1.4"
     },
     "engines": {
-        "node": ">=10.16.0"
+        "node": "^12.20.0 || >=14.17.0",
+        "yarn": ">=1.16.0"
     },
     "publishConfig": {
         "access": "public",

packages/data-types/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/data-types",
     "displayName": "Data Types",
-    "version": "0.28.1",
+    "version": "0.29.0",
     "description": "A library for defining the data structures and mapping",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme",
     "bugs": {
@@ -28,8 +28,8 @@
         "test:watch": "ts-scripts test --watch . --"
     },
     "dependencies": {
-        "@terascope/types": "^0.8.0",
-        "@terascope/utils": "^0.38.1",
+        "@terascope/types": "^0.9.0",
+        "@terascope/utils": "^0.39.0",
         "graphql": "^15.5.0",
         "lodash": "^4.17.21",
         "yargs": "^16.2.0"
@@ -39,7 +39,8 @@
         "@types/yargs": "^16.0.1"
     },
     "engines": {
-        "node": ">=10.16.0"
+        "node": "^12.20.0 || >=14.17.0",
+        "yarn": ">=1.16.0"
     },
     "publishConfig": {
         "access": "public",