diff --git a/.gitignore b/.gitignore index b3911d0f8d0c2..b8adcf4508db2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ /.es .DS_Store .node_binaries +.native_modules node_modules !/src/dev/npm/integration_tests/__fixtures__/fixture1/node_modules !/src/dev/notice/__fixtures__/node_modules diff --git a/package.json b/package.json index 46bafdaa36df0..4d5fdebf9ad0c 100644 --- a/package.json +++ b/package.json @@ -233,6 +233,7 @@ "pug": "^2.0.4", "query-string": "5.1.1", "raw-loader": "3.1.0", + "re2": "1.14.0", "react": "^16.12.0", "react-color": "^2.13.8", "react-dom": "^16.12.0", diff --git a/src/dev/build/build_distributables.js b/src/dev/build/build_distributables.js index 66f0c0355c2d9..2ea71fa2c1d33 100644 --- a/src/dev/build/build_distributables.js +++ b/src/dev/build/build_distributables.js @@ -47,6 +47,7 @@ import { InstallDependenciesTask, BuildKibanaPlatformPluginsTask, OptimizeBuildTask, + PatchNativeModulesTask, RemovePackageJsonDepsTask, RemoveWorkspacesTask, TranspileBabelTask, @@ -136,6 +137,7 @@ export async function buildDistributables(options) { * directories and perform platform-specific steps */ await run(CreateArchivesSourcesTask); + await run(PatchNativeModulesTask); await run(CleanExtraBinScriptsTask); await run(CleanExtraBrowsersTask); await run(CleanNodeBuildsTask); diff --git a/src/dev/build/tasks/nodejs/__tests__/download.js b/src/dev/build/lib/__tests__/download.js similarity index 100% rename from src/dev/build/tasks/nodejs/__tests__/download.js rename to src/dev/build/lib/__tests__/download.js diff --git a/src/dev/build/lib/__tests__/fixtures/foo.txt.gz b/src/dev/build/lib/__tests__/fixtures/foo.txt.gz new file mode 100644 index 0000000000000..46fef5a6af78c Binary files /dev/null and b/src/dev/build/lib/__tests__/fixtures/foo.txt.gz differ diff --git a/src/dev/build/lib/__tests__/fs.js b/src/dev/build/lib/__tests__/fs.js index 0b2db4c538fb8..bf7596b012f79 100644 --- a/src/dev/build/lib/__tests__/fs.js +++ b/src/dev/build/lib/__tests__/fs.js @@ -23,11 +23,12 @@ import { chmodSync, statSync } from 'fs'; import del from 'del'; import expect from '@kbn/expect'; -import { mkdirp, write, read, getChildPaths, copyAll, getFileHash, untar } from '../fs'; +import { mkdirp, write, read, getChildPaths, copyAll, getFileHash, untar, gunzip } from '../fs'; const TMP = resolve(__dirname, '__tmp__'); const FIXTURES = resolve(__dirname, 'fixtures'); const FOO_TAR_PATH = resolve(FIXTURES, 'foo_dir.tar.gz'); +const FOO_GZIP_PATH = resolve(FIXTURES, 'foo.txt.gz'); const BAR_TXT_PATH = resolve(FIXTURES, 'foo_dir/bar.txt'); const WORLD_EXECUTABLE = resolve(FIXTURES, 'bin/world_executable'); @@ -323,4 +324,39 @@ describe('dev/build/lib/fs', () => { expect(await read(resolve(destination, 'foo/foo.txt'))).to.be('foo\n'); }); }); + + describe('gunzip()', () => { + it('rejects if source path is not absolute', async () => { + try { + await gunzip('foo/bar', '**/*', __dirname); + throw new Error('Expected gunzip() to reject'); + } catch (error) { + assertNonAbsoluteError(error); + } + }); + + it('rejects if destination path is not absolute', async () => { + try { + await gunzip(__dirname, '**/*', 'foo/bar'); + throw new Error('Expected gunzip() to reject'); + } catch (error) { + assertNonAbsoluteError(error); + } + }); + + it('rejects if neither path is not absolute', async () => { + try { + await gunzip('foo/bar', '**/*', 'foo/bar'); + throw new Error('Expected gunzip() to reject'); + } catch (error) { + assertNonAbsoluteError(error); + } + }); + + it('extracts gzip from source into destination, creating destination if necessary', async () => { + const destination = resolve(TMP, 'z/y/x/v/u/t/foo.txt'); + await gunzip(FOO_GZIP_PATH, destination); + expect(await read(resolve(destination))).to.be('foo\n'); + }); + }); }); diff --git a/src/dev/build/tasks/nodejs/download.js b/src/dev/build/lib/download.js similarity index 98% rename from src/dev/build/tasks/nodejs/download.js rename to src/dev/build/lib/download.js index fb3294e2d1221..fbd2d47ff7b06 100644 --- a/src/dev/build/tasks/nodejs/download.js +++ b/src/dev/build/lib/download.js @@ -24,7 +24,7 @@ import chalk from 'chalk'; import { createHash } from 'crypto'; import Axios from 'axios'; -import { mkdirp } from '../../lib'; +import { mkdirp } from './fs'; function tryUnlink(path) { try { diff --git a/src/dev/build/lib/fs.js b/src/dev/build/lib/fs.js index 864a07e837c3f..b905f40d0de1e 100644 --- a/src/dev/build/lib/fs.js +++ b/src/dev/build/lib/fs.js @@ -195,6 +195,19 @@ export async function untar(source, destination, extractOptions = {}) { ]); } +export async function gunzip(source, destination) { + assertAbsolute(source); + assertAbsolute(destination); + + await mkdirAsync(dirname(destination), { recursive: true }); + + await createPromiseFromStreams([ + fs.createReadStream(source), + createGunzip(), + fs.createWriteStream(destination), + ]); +} + export async function compress(type, options = {}, source, destination) { const output = fs.createWriteStream(destination); const archive = archiver(type, options.archiverOptions); diff --git a/src/dev/build/lib/index.js b/src/dev/build/lib/index.js index afebd090d797d..6540db6f37a72 100644 --- a/src/dev/build/lib/index.js +++ b/src/dev/build/lib/index.js @@ -28,10 +28,12 @@ export { copyAll, getFileHash, untar, + gunzip, deleteAll, deleteEmptyFolders, compress, isFileAccessible, } from './fs'; +export { download } from './download'; export { scanDelete } from './scan_delete'; export { scanCopy } from './scan_copy'; diff --git a/src/dev/build/tasks/index.js b/src/dev/build/tasks/index.js index bafb5a2fe115e..be675b4aa6ca4 100644 --- a/src/dev/build/tasks/index.js +++ b/src/dev/build/tasks/index.js @@ -33,6 +33,7 @@ export * from './nodejs_modules'; export * from './notice_file_task'; export * from './optimize_task'; export * from './os_packages'; +export * from './patch_native_modules_task'; export * from './transpile_babel_task'; export * from './transpile_scss_task'; export * from './verify_env_task'; diff --git a/src/dev/build/tasks/nodejs/__tests__/download_node_builds_task.js b/src/dev/build/tasks/nodejs/__tests__/download_node_builds_task.js index 9357735e3f5a3..c1764d06b43b3 100644 --- a/src/dev/build/tasks/nodejs/__tests__/download_node_builds_task.js +++ b/src/dev/build/tasks/nodejs/__tests__/download_node_builds_task.js @@ -22,7 +22,7 @@ import expect from '@kbn/expect'; import * as NodeShasumsNS from '../node_shasums'; import * as NodeDownloadInfoNS from '../node_download_info'; -import * as DownloadNS from '../download'; +import * as DownloadNS from '../../../lib/download'; // sinon can't stub '../../../lib' properly import { DownloadNodeBuildsTask } from '../download_node_builds_task'; describe('src/dev/build/tasks/nodejs/download_node_builds_task', () => { diff --git a/src/dev/build/tasks/nodejs/download_node_builds_task.js b/src/dev/build/tasks/nodejs/download_node_builds_task.js index 86ddb0506f972..c0907e6c42a97 100644 --- a/src/dev/build/tasks/nodejs/download_node_builds_task.js +++ b/src/dev/build/tasks/nodejs/download_node_builds_task.js @@ -17,7 +17,7 @@ * under the License. */ -import { download } from './download'; +import { download } from '../../lib'; import { getNodeShasums } from './node_shasums'; import { getNodeDownloadInfo } from './node_download_info'; diff --git a/src/dev/build/tasks/patch_native_modules_task.js b/src/dev/build/tasks/patch_native_modules_task.js new file mode 100644 index 0000000000000..fba33442fad10 --- /dev/null +++ b/src/dev/build/tasks/patch_native_modules_task.js @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import fs from 'fs'; +import path from 'path'; +import util from 'util'; +import { deleteAll, download, gunzip, untar } from '../lib'; + +const DOWNLOAD_DIRECTORY = '.native_modules'; + +const packages = [ + { + name: 're2', + version: '1.14.0', + destinationPath: 'node_modules/re2/build/Release/re2.node', + extractMethod: 'gunzip', + archives: { + darwin: { + url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/darwin-x64-64.gz', + sha256: '54c8386cb7cd53895cf379522114bfe82378e300e127e58d392ddd40a77e396f', + }, + linux: { + url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/linux-x64-64.gz', + sha256: 'f54f059035e71a7ccb3fa201080e260c41d228d13a8247974b4bb157691b6757', + }, + windows: { + url: 'https://github.com/uhop/node-re2/releases/download/1.14.0/win32-x64-64.gz', + sha256: 'de708446a8b802f4634c2cfef097c2625a2811fdcd8133dfd7b7c485f966caa9', + }, + }, + }, +]; + +async function getInstalledVersion(config, packageName) { + const packageJSONPath = config.resolveFromRepo( + path.join('node_modules', packageName, 'package.json') + ); + const buffer = await util.promisify(fs.readFile)(packageJSONPath); + const packageJSON = JSON.parse(buffer); + return packageJSON.version; +} + +async function patchModule(config, log, build, platform, pkg) { + const installedVersion = await getInstalledVersion(config, pkg.name); + if (installedVersion !== pkg.version) { + throw new Error( + `Can't patch ${pkg.name}'s native module, we were expecting version ${pkg.version} and found ${installedVersion}` + ); + } + const platformName = platform.getName(); + const archive = pkg.archives[platformName]; + const archiveName = path.basename(archive.url); + const downloadPath = config.resolveFromRepo(DOWNLOAD_DIRECTORY, pkg.name, archiveName); + const extractPath = build.resolvePathForPlatform(platform, pkg.destinationPath); + log.debug(`Patching ${pkg.name} binaries from ${archive.url} to ${extractPath}`); + + await deleteAll([extractPath], log); + await download({ + log, + url: archive.url, + destination: downloadPath, + sha256: archive.sha256, + retries: 3, + }); + switch (pkg.extractMethod) { + case 'gunzip': + await gunzip(downloadPath, extractPath); + break; + case 'untar': + await untar(downloadPath, extractPath); + break; + default: + throw new Error(`Extract method of ${pkg.extractMethod} is not supported`); + } +} + +export const PatchNativeModulesTask = { + description: 'Patching platform-specific native modules', + async run(config, log, build) { + for (const pkg of packages) { + await Promise.all( + config.getTargetPlatforms().map(async (platform) => { + await patchModule(config, log, build, platform, pkg); + }) + ); + } + }, +}; diff --git a/src/dev/jest/config.js b/src/dev/jest/config.js index 1fbcc6e5f2a6a..ded7606050ba9 100644 --- a/src/dev/jest/config.js +++ b/src/dev/jest/config.js @@ -77,7 +77,7 @@ export default { ], coverageDirectory: '/target/kibana-coverage/jest', coverageReporters: ['html', 'text'], - moduleFileExtensions: ['js', 'json', 'ts', 'tsx'], + moduleFileExtensions: ['js', 'json', 'ts', 'tsx', 'node'], modulePathIgnorePatterns: ['__fixtures__/', 'target/'], testMatch: ['**/*.test.{js,ts,tsx}'], testPathIgnorePatterns: [ diff --git a/src/plugins/vis_type_timelion/server/series_functions/label.js b/src/plugins/vis_type_timelion/server/series_functions/label.js index 4d9a26c46aaaf..b5282967a62e0 100644 --- a/src/plugins/vis_type_timelion/server/series_functions/label.js +++ b/src/plugins/vis_type_timelion/server/series_functions/label.js @@ -51,7 +51,10 @@ export default new Chainable('label', { const config = args.byName; return alter(args, function (eachSeries) { if (config.regex) { - eachSeries.label = eachSeries.label.replace(new RegExp(config.regex), config.label); + // not using a standard `import` so that if there's an issue with the re2 native module + // that it doesn't prevent Kibana from starting up and we only have an issue using Timelion labels + const RE2 = require('re2'); + eachSeries.label = eachSeries.label.replace(new RE2(config.regex), config.label); } else { eachSeries.label = config.label; } diff --git a/x-pack/dev-tools/jest/create_jest_config.js b/x-pack/dev-tools/jest/create_jest_config.js index c360711e37e27..52a6c83124f36 100644 --- a/x-pack/dev-tools/jest/create_jest_config.js +++ b/x-pack/dev-tools/jest/create_jest_config.js @@ -9,7 +9,7 @@ export function createJestConfig({ kibanaDirectory, rootDir, xPackKibanaDirector return { rootDir, roots: ['/plugins', '/legacy/plugins', '/legacy/server'], - moduleFileExtensions: ['js', 'json', 'ts', 'tsx'], + moduleFileExtensions: ['js', 'json', 'ts', 'tsx', 'node'], moduleNameMapper: { '@elastic/eui$': `${kibanaDirectory}/node_modules/@elastic/eui/test-env`, '@elastic/eui/lib/(.*)?': `${kibanaDirectory}/node_modules/@elastic/eui/test-env/$1`, diff --git a/x-pack/test_utils/jest/config.js b/x-pack/test_utils/jest/config.js index deee585b91fe4..f36099bd1f347 100644 --- a/x-pack/test_utils/jest/config.js +++ b/x-pack/test_utils/jest/config.js @@ -29,7 +29,7 @@ export default { ], coverageDirectory: '/../target/kibana-coverage/jest', coverageReporters: ['html'], - moduleFileExtensions: ['js', 'json', 'ts', 'tsx'], + moduleFileExtensions: ['js', 'json', 'ts', 'tsx', 'node'], modulePathIgnorePatterns: ['__fixtures__/', 'target/'], testMatch: ['**/*.test.{js,ts,tsx}'], testPathIgnorePatterns: [ diff --git a/yarn.lock b/yarn.lock index a0ab45e57eda9..1aa77794947be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12975,6 +12975,11 @@ env-paths@^1.0.0: resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-1.0.0.tgz#4168133b42bb05c38a35b1ae4397c8298ab369e0" integrity sha1-QWgTO0K7BcOKNbGuQ5fIKYqzaeA= +env-paths@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.0.tgz#cdca557dc009152917d6166e2febe1f039685e43" + integrity sha512-6u0VYSCo/OW6IoD5WCLLy9JUGARbamfSavcNXry/eu8aHVFei6CD3Sw+VGX5alea1i9pgPHW0mbu6Xj0uBh7gA== + env-variable@0.0.x: version "0.0.5" resolved "https://registry.yarnpkg.com/env-variable/-/env-variable-0.0.5.tgz#913dd830bef11e96a039c038d4130604eba37f88" @@ -21923,6 +21928,11 @@ nan@^2.12.1, nan@^2.13.2: resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c" integrity sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg== +nan@^2.14.1: + version "2.14.1" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.1.tgz#d7be34dfa3105b91494c3147089315eff8874b01" + integrity sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw== + nano-css@^5.2.1: version "5.2.1" resolved "https://registry.yarnpkg.com/nano-css/-/nano-css-5.2.1.tgz#73b8470fa40b028a134d3393ae36bbb34b9fa332" @@ -22189,6 +22199,23 @@ node-gyp@^3.8.0: tar "^2.0.0" which "1" +node-gyp@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-6.1.0.tgz#64e31c61a4695ad304c1d5b82cf6b7c79cc79f3f" + integrity sha512-h4A2zDlOujeeaaTx06r4Vy+8MZ1679lU+wbCKDS4ZtvY2A37DESo37oejIw0mtmR3+rvNwts5B6Kpt1KrNYdNw== + dependencies: + env-paths "^2.2.0" + glob "^7.1.4" + graceful-fs "^4.2.2" + mkdirp "^0.5.1" + nopt "^4.0.1" + npmlog "^4.1.2" + request "^2.88.0" + rimraf "^2.6.3" + semver "^5.7.1" + tar "^4.4.12" + which "^1.3.1" + node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" @@ -22386,6 +22413,14 @@ nopt@^2.2.0: dependencies: abbrev "1" +nopt@^4.0.1: + version "4.0.3" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48" + integrity sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg== + dependencies: + abbrev "1" + osenv "^0.1.4" + nopt@~1.0.10: version "1.0.10" resolved "https://registry.yarnpkg.com/nopt/-/nopt-1.0.10.tgz#6ddd21bd2a31417b92727dd585f8a6f37608ebee" @@ -23080,7 +23115,7 @@ os-tmpdir@^1.0.0, os-tmpdir@~1.0.1, os-tmpdir@~1.0.2: resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= -osenv@0, osenv@^0.1.0: +osenv@0, osenv@^0.1.0, osenv@^0.1.4: version "0.1.5" resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410" integrity sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g== @@ -24964,6 +24999,14 @@ re-resizable@^6.1.1: dependencies: fast-memoize "^2.5.1" +re2@1.14.0: + version "1.14.0" + resolved "https://registry.yarnpkg.com/re2/-/re2-1.14.0.tgz#727076590acfe868cf04e115a3a3f6c373ddd63b" + integrity sha512-TYogJmzni8zNVaw4gNOVORRTUaggLZwnMhJoTD0POKeACEoCxTWa9BAYehRnh3S1JUXIMEfcEUa7piiGEn71Zg== + dependencies: + nan "^2.14.1" + node-gyp "^6.1.0" + react-ace@^5.5.0: version "5.10.0" resolved "https://registry.yarnpkg.com/react-ace/-/react-ace-5.10.0.tgz#e328b37ac52759f700be5afdb86ada2f5ec84c5e" @@ -29405,7 +29448,7 @@ tar-stream@^2.1.0: inherits "^2.0.3" readable-stream "^3.1.1" -tar@4.4.13: +tar@4.4.13, tar@^4.4.12: version "4.4.13" resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525" integrity sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==