From 955972b2b5110d582c9547a2c7868155435ea908 Mon Sep 17 00:00:00 2001 From: ppisljar Date: Mon, 11 Jul 2016 21:17:08 +0200 Subject: [PATCH] fix #7136 - check SHA of downloaded node binaries --- package.json | 1 + tasks/build/download_node_builds.js | 193 ++++++++++++++++++---------- tasks/build/index.js | 4 +- tasks/config/platforms.js | 4 +- 4 files changed, 133 insertions(+), 69 deletions(-) diff --git a/package.json b/package.json index 24829b260106f..b5425331a79c5 100644 --- a/package.json +++ b/package.json @@ -97,6 +97,7 @@ "boom": "2.8.0", "brace": "0.5.1", "bunyan": "1.7.1", + "check-hash": "1.0.1", "commander": "2.8.1", "css-loader": "0.17.0", "d3": "3.5.6", diff --git a/tasks/build/download_node_builds.js b/tasks/build/download_node_builds.js index cb74b55cec446..09f1a1e7f099a 100644 --- a/tasks/build/download_node_builds.js +++ b/tasks/build/download_node_builds.js @@ -1,84 +1,145 @@ -module.exports = function (grunt) { - let { map, fromNode } = require('bluebird'); - let { resolve } = require('path'); - let { pluck } = require('lodash'); - let { createWriteStream } = require('fs'); - let { createGunzip } = require('zlib'); - let { Extract } = require('tar'); - let { rename } = require('fs'); - let wreck = require('wreck'); - - let platforms = grunt.config.get('platforms'); - let activeDownloads = []; - - let start = async (platform) => { - let finalDir = platform.nodeDir; - let downloadDir = `${finalDir}.temp`; +import { Promise, map, fromNode, promisify } from 'bluebird'; +import { resolve, basename, dirname, join } from 'path'; +import { createReadStream, createWriteStream, writeFile } from 'fs'; +import { createGunzip } from 'zlib'; +import { Extract } from 'tar'; +import { fromFile } from 'check-hash'; +import wreck from 'wreck'; - if (grunt.file.isDir(platform.nodeDir)) { - grunt.log.ok(`${platform.name} exists`); - return; - } +const wreckGetAsync = promisify(wreck.get, wreck); +const checkHashFromFileAsync = promisify(fromFile); +const writeFileAsync = promisify(writeFile); - let resp = await fromNode(cb => { - let req = wreck.request('GET', platform.nodeUrl, null, function (err, resp) { - if (err) { - return cb(err); - } +export default function downloadNodeBuilds(grunt) { + const platforms = grunt.config.get('platforms'); + const downloadLimit = 3; - if (resp.statusCode !== 200) { - return cb(new Error(`${platform.nodeUrl} failed with a ${resp.statusCode} response`)); - } + let shaSums = {}; + const getShaSums = () => { + const nodeVersion = grunt.config.get('nodeVersion'); + const shaSumsUri = `https://nodejs.org/dist/v${nodeVersion}/SHASUMS256.txt`; - return cb(null, resp); + return wreckGetAsync(shaSumsUri).then(([resp, payload]) => { + if (resp.statusCode !== 200) { + throw new Error(`${shaSumsUri} failed with a ${resp.statusCode} response`); + } + payload + .toString('utf8') + .split('\n') + .forEach(line => { + const [sha, platform] = line.split(' '); + shaSums[platform] = sha; }); + grunt.log.ok(`sha sums download complete`); }); + }; + + const checkShaSum = (platform) => { + const file = basename(platform.nodeUrl); + const downloadDir = join(platform.nodeDir, '..'); + const filePath = resolve(downloadDir, file); + const expected = { + hash: 'sha256', + expected: platform.win ? shaSums[basename(dirname(platform.nodeUrl)) + '/' + file] : shaSums[file] + }; - // use an async iife to store promise for download - // then store platform in active downloads list - // which we will read from in the finish task - platform.downloadPromise = (async function () { - grunt.file.mkdir(downloadDir); - - if (platform.win) { - await fromNode(cb => { - resp - .pipe(createWriteStream(resolve(downloadDir, 'node.exe'))) - .on('error', cb) - .on('finish', cb); - }); + if (!grunt.file.isFile(filePath)) { + return false; + } + + return checkHashFromFileAsync(filePath, expected).then(([passed, actual]) => { + if (passed) { + grunt.log.ok(`${platform.name} shasum verified`); } else { - await fromNode(cb => { - resp - .pipe(createGunzip()) - .on('error', cb) - .pipe(new Extract({ path: downloadDir, strip: 1 })) - .on('error', cb) - .on('end', cb); - }); + grunt.log.error(`${platform.name} shasum check failed`); + } + return passed; + }); + }; + + const getNodeBuild = (platform) => { + const downloadDir = join(platform.nodeDir, '..'); + const file = basename(platform.nodeUrl); + const filePath = resolve(downloadDir, file); + + if (grunt.file.isFile(filePath)) { + grunt.file.delete(filePath); + } + + grunt.log.ok(`downloading ${platform.name}`); + + return wreckGetAsync(platform.nodeUrl) + .then(([resp, payload]) => { + if (resp.statusCode !== 200) { + throw new Error(`${platform.nodeUrl} failed with a ${resp.statusCode} response`); } + return payload; + }) + .then(payload => writeFileAsync(filePath, payload)); - await fromNode(cb => { - rename(downloadDir, finalDir, cb); - }); - }()); + }; - activeDownloads.push(platform); + const start = async (platform) => { + let downloadCounter = 0; + let isDownloadValid = await checkShaSum(platform); - var bytes = parseInt(resp.headers['content-length'], 10) || 'unknown number of'; - var mb = ((bytes / 1024) / 1024).toFixed(2); - grunt.log.ok(`downloading ${platform.name} - ${mb} mb`); + if (isDownloadValid) return; + + grunt.log.ok('starting download ...'); + while (!isDownloadValid && (downloadCounter < downloadLimit)) { + await getNodeBuild(platform); + isDownloadValid = await checkShaSum(platform); + ++downloadCounter; + } + + if (!isDownloadValid) { + throw new Error(`${platform.name} download failed`); + } + + grunt.log.ok(`download of ${platform.name} completed successfully`); }; - grunt.registerTask('_build:downloadNodeBuilds:start', function () { - map(platforms, start).nodeify(this.async()); + grunt.registerTask('_build:downloadNodeBuilds', function () { + const done = this.async(); + getShaSums() + .then(() => map(platforms, start)) + .nodeify(done); }); - grunt.registerTask('_build:downloadNodeBuilds:finish', function () { - map(activeDownloads, async (platform) => { - await platform.downloadPromise; - grunt.log.ok(`${platform.name} download complete`); - }) - .nodeify(this.async()); + const extractNodeBuild = async (platform) => { + const file = basename(platform.nodeUrl); + const downloadDir = join(platform.nodeDir, '..'); + const filePath = resolve(downloadDir, file); + + return new Promise((resolve, reject) => { + createReadStream(filePath) + .pipe(createGunzip()) + .on('error', reject) + .pipe(new Extract({path: platform.nodeDir, strip: 1})) + .on('error', reject) + .on('end', resolve); + }); + }; + + const extract = async(platform) => { + const file = basename(platform.nodeUrl); + const downloadDir = join(platform.nodeDir, '..'); + const filePath = resolve(downloadDir, file); + + if (grunt.file.isDir(platform.nodeDir)) { + grunt.file.delete(platform.nodeDir); + } + + if (platform.win) { + grunt.file.mkdir(platform.nodeDir); + grunt.file.copy(filePath, resolve(platform.nodeDir, file)); + } else { + await extractNodeBuild(platform); + } + }; + + grunt.registerTask('_build:extractNodeBuilds', function () { + map(platforms, extract).nodeify(this.async()); }); + }; diff --git a/tasks/build/index.js b/tasks/build/index.js index 4de6a1407a364..1fa50df44073c 100644 --- a/tasks/build/index.js +++ b/tasks/build/index.js @@ -5,7 +5,8 @@ module.exports = function (grunt) { grunt.task.run(flatten([ 'clean:build', 'clean:target', - '_build:downloadNodeBuilds:start', + '_build:downloadNodeBuilds', + '_build:extractNodeBuilds', 'copy:devSource', 'babel:build', '_build:babelOptions', @@ -21,7 +22,6 @@ module.exports = function (grunt) { 'clean:deepModules', 'run:optimizeBuild', 'stop:optimizeBuild', - '_build:downloadNodeBuilds:finish', '_build:versionedLinks', '_build:osShellScripts', grunt.option('skip-archives') ? [] : ['_build:archives'], diff --git a/tasks/config/platforms.js b/tasks/config/platforms.js index 5fa47841bc9bd..8ff8c808a4a5e 100644 --- a/tasks/config/platforms.js +++ b/tasks/config/platforms.js @@ -23,6 +23,8 @@ module.exports = function (grunt) { ? baseName.replace('-x64', '-x86_64') : baseName; + let nodeShaSums = `${baseUri}/SHASUMS256.txt`; + let buildName = `kibana-${version}-${name}`; let buildDir = resolve(rootPath, `build/${buildName}`); @@ -49,7 +51,7 @@ module.exports = function (grunt) { } return { name, win, - nodeUrl, nodeDir, + nodeUrl, nodeDir, nodeShaSums, buildName, buildDir, tarName, tarPath, zipName, zipPath,