diff --git a/src/dev/build/lib/download.ts b/src/dev/build/lib/download.ts index ce2bdbd33e8c..c67407095b37 100644 --- a/src/dev/build/lib/download.ts +++ b/src/dev/build/lib/download.ts @@ -8,11 +8,12 @@ import { openSync, writeSync, unlinkSync, closeSync } from 'fs'; import { dirname } from 'path'; +import { setTimeout } from 'timers/promises'; import chalk from 'chalk'; import { createHash } from 'crypto'; import Axios from 'axios'; -import { ToolingLog } from '@kbn/dev-utils'; +import { ToolingLog, isAxiosResponseError } from '@kbn/dev-utils'; // https://github.com/axios/axios/tree/ffea03453f77a8176c51554d5f6c3c6829294649/lib/adapters // @ts-expect-error untyped internal module used to prevent axios from using xhr adapter in tests @@ -30,80 +31,151 @@ function tryUnlink(path: string) { } } -interface DownloadOptions { +interface DownloadToDiskOptions { log: ToolingLog; url: string; destination: string; - sha256: string; - retries?: number; + shaChecksum: string; + shaAlgorithm: string; + maxAttempts?: number; + retryDelaySecMultiplier?: number; } -export async function download(options: DownloadOptions): Promise { - const { log, url, destination, sha256, retries = 0 } = options; +export async function downloadToDisk({ + log, + url, + destination, + shaChecksum, + shaAlgorithm, + maxAttempts = 1, + retryDelaySecMultiplier = 5, +}: DownloadToDiskOptions) { + if (!shaChecksum) { + throw new Error(`${shaAlgorithm} checksum of ${url} not provided, refusing to download.`); + } - if (!sha256) { - throw new Error(`sha256 checksum of ${url} not provided, refusing to download.`); + if (maxAttempts < 1) { + throw new Error(`[maxAttempts=${maxAttempts}] must be >= 1`); } - // mkdirp and open file outside of try/catch, we don't retry for those errors - await mkdirp(dirname(destination)); - const fileHandle = openSync(destination, 'w'); + let attempt = 0; + while (true) { + attempt += 1; - let error; - try { - log.debug(`Attempting download of ${url}`, chalk.dim(sha256)); + // mkdirp and open file outside of try/catch, we don't retry for those errors + await mkdirp(dirname(destination)); + const fileHandle = openSync(destination, 'w'); + + let error; + try { + log.debug( + `[${attempt}/${maxAttempts}] Attempting download of ${url}`, + chalk.dim(shaAlgorithm) + ); - const response = await Axios.request({ - url, - responseType: 'stream', - adapter: AxiosHttpAdapter, - }); + const response = await Axios.request({ + url, + responseType: 'stream', + adapter: AxiosHttpAdapter, + }); - if (response.status !== 200) { - throw new Error(`Unexpected status code ${response.status} when downloading ${url}`); - } + if (response.status !== 200) { + throw new Error(`Unexpected status code ${response.status} when downloading ${url}`); + } - const hash = createHash('sha256'); - await new Promise((resolve, reject) => { - response.data.on('data', (chunk: Buffer) => { - hash.update(chunk); - writeSync(fileHandle, chunk); + const hash = createHash(shaAlgorithm); + await new Promise((resolve, reject) => { + response.data.on('data', (chunk: Buffer) => { + hash.update(chunk); + writeSync(fileHandle, chunk); + }); + + response.data.on('error', reject); + response.data.on('end', resolve); }); - response.data.on('error', reject); - response.data.on('end', resolve); - }); + const downloadedSha = hash.digest('hex'); + if (downloadedSha !== shaChecksum) { + throw new Error( + `Downloaded checksum ${downloadedSha} does not match the expected ${shaAlgorithm} checksum.` + ); + } + } catch (_error) { + error = _error; + } finally { + closeSync(fileHandle); + } - const downloadedSha256 = hash.digest('hex'); - if (downloadedSha256 !== sha256) { - throw new Error( - `Downloaded checksum ${downloadedSha256} does not match the expected sha256 checksum.` - ); + if (!error) { + log.debug(`Downloaded ${url} and verified checksum`); + return; } - } catch (_error) { - error = _error; - } finally { - closeSync(fileHandle); - } - if (!error) { - log.debug(`Downloaded ${url} and verified checksum`); - return; - } + log.debug(`Download failed: ${error.message}`); - log.debug(`Download failed: ${error.message}`); + // cleanup downloaded data and log error + log.debug(`Deleting downloaded data at ${destination}`); + tryUnlink(destination); - // cleanup downloaded data and log error - log.debug(`Deleting downloaded data at ${destination}`); - tryUnlink(destination); + // retry if we have retries left + if (attempt < maxAttempts) { + const sec = attempt * retryDelaySecMultiplier; + log.info(`Retrying in ${sec} seconds`); + await setTimeout(sec * 1000); + continue; + } - // retry if we have retries left - if (retries > 0) { - log.debug(`Retrying - ${retries} attempt remaining`); - return await download({ - ...options, - retries: retries - 1, - }); + throw error; } +} - throw error; +interface DownloadToStringOptions { + log: ToolingLog; + url: string; + expectStatus?: number; + maxAttempts?: number; + retryDelaySecMultiplier?: number; +} +export async function downloadToString({ + log, + url, + expectStatus, + maxAttempts = 3, + retryDelaySecMultiplier = 5, +}: DownloadToStringOptions) { + let attempt = 0; + while (true) { + try { + attempt += 1; + log.debug(`[${attempt}/${maxAttempts}] Attempting download to string of [${url}]`); + + const resp = await Axios.request({ + url, + method: 'GET', + adapter: AxiosHttpAdapter, + responseType: 'text', + validateStatus: !expectStatus ? undefined : (status) => status === expectStatus, + }); + + log.success(`Downloaded [${url}]`); + return resp.data; + } catch (error) { + log.warning(`Download failed: ${error.message}`); + if (isAxiosResponseError(error)) { + log.debug( + `[${error.response.status}/${error.response.statusText}] response: ${error.response.data}` + ); + } else { + log.debug('received no response'); + } + + if ((maxAttempts ?? 3) > attempt) { + const sec = (retryDelaySecMultiplier ?? 5) * attempt; + log.info(`Retrying in ${sec} seconds`); + await setTimeout(sec * 1000); + continue; + } + + throw error; + } + } } diff --git a/src/dev/build/lib/integration_tests/download.test.ts b/src/dev/build/lib/integration_tests/download.test.ts index 9003e678e98a..7f885fdc02d2 100644 --- a/src/dev/build/lib/integration_tests/download.test.ts +++ b/src/dev/build/lib/integration_tests/download.test.ts @@ -13,14 +13,25 @@ import { readFileSync } from 'fs'; import del from 'del'; import { CI_PARALLEL_PROCESS_PREFIX } from '@kbn/test'; -import { ToolingLog } from '@kbn/dev-utils'; +import { + ToolingLog, + ToolingLogCollectingWriter, + createStripAnsiSerializer, + createReplaceSerializer, +} from '@kbn/dev-utils'; import { mkdirp } from '../fs'; -import { download } from '../download'; +import { downloadToDisk, downloadToString } from '../download'; const TMP_DIR = join(tmpdir(), CI_PARALLEL_PROCESS_PREFIX, 'download-js-test-tmp-dir'); const TMP_DESTINATION = join(TMP_DIR, '__tmp_download_js_test_file__'); +expect.addSnapshotSerializer(createStripAnsiSerializer()); +expect.addSnapshotSerializer(createReplaceSerializer(TMP_DIR, 'TMP_DIR')); +expect.addSnapshotSerializer( + createReplaceSerializer(/http:\/\/localhost:\d+\//g, 'TEST_SERVER_URL') +); + beforeEach(async () => { await del(TMP_DIR, { force: true }); await mkdirp(TMP_DIR); @@ -31,12 +42,11 @@ afterEach(async () => { await del(TMP_DIR, { force: true }); }); -const onLogLine = jest.fn(); -const log = new ToolingLog({ - level: 'verbose', - writeTo: { - write: onLogLine, - }, +const logWritter = new ToolingLogCollectingWriter('verbose'); +const log = new ToolingLog(); +log.setWriters([logWritter]); +afterEach(() => { + logWritter.messages.length = 0; }); type Handler = (req: IncomingMessage, res: ServerResponse) => void; @@ -54,18 +64,14 @@ const sendErrorHandler: Handler = (req, res) => { }; let serverUrl: string; -let nextHandler: Handler | null = null; +const handlers: Handler[] = []; const server = createServer((req, res) => { - if (!nextHandler) { - nextHandler = sendErrorHandler; - } - - const handler = nextHandler; - nextHandler = null; - handler(req, res); + (handlers.shift() ?? sendErrorHandler)(req, res); }); -afterEach(() => (nextHandler = null)); +afterEach(() => { + handlers.length = 0; +}); beforeAll(async () => { await Promise.race([ @@ -87,131 +93,224 @@ afterAll(async () => { server.close(); }); -it('downloads from URL and checks that content matches sha256', async () => { - nextHandler = createSendHandler('foo'); - await download({ - log, - url: serverUrl, - destination: TMP_DESTINATION, - sha256: FOO_SHA256, - }); - expect(readFileSync(TMP_DESTINATION, 'utf8')).toBe('foo'); -}); - -it('rejects and deletes destination if sha256 does not match', async () => { - nextHandler = createSendHandler('foo'); - - try { - await download({ +describe('downloadToDisk', () => { + it('downloads from URL and checks that content matches sha256', async () => { + handlers.push(createSendHandler('foo')); + await downloadToDisk({ log, url: serverUrl, destination: TMP_DESTINATION, - sha256: 'bar', + shaChecksum: FOO_SHA256, + shaAlgorithm: 'sha256', }); - throw new Error('Expected download() to reject'); - } catch (error) { - expect(error).toHaveProperty( - 'message', - expect.stringContaining('does not match the expected sha256 checksum') - ); - } - - try { - readFileSync(TMP_DESTINATION); - throw new Error('Expected download to be deleted'); - } catch (error) { - expect(error).toHaveProperty('code', 'ENOENT'); - } -}); + expect(readFileSync(TMP_DESTINATION, 'utf8')).toBe('foo'); + }); -describe('reties download retries: number of times', () => { - it('resolves if retries = 1 and first attempt fails', async () => { - let reqCount = 0; - nextHandler = function sequenceHandler(req, res) { - switch (++reqCount) { - case 1: - nextHandler = sequenceHandler; - return sendErrorHandler(req, res); - default: - return createSendHandler('foo')(req, res); - } - }; + it('rejects and deletes destination if sha256 does not match', async () => { + handlers.push(createSendHandler('foo')); - await download({ + const promise = downloadToDisk({ log, url: serverUrl, destination: TMP_DESTINATION, - sha256: FOO_SHA256, - retries: 2, + shaChecksum: 'bar', + shaAlgorithm: 'sha256', }); + await expect(promise).rejects.toMatchInlineSnapshot( + `[Error: Downloaded checksum 2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae does not match the expected sha256 checksum.]` + ); - expect(readFileSync(TMP_DESTINATION, 'utf8')).toBe('foo'); + try { + readFileSync(TMP_DESTINATION); + throw new Error('Expected download to be deleted'); + } catch (error) { + expect(error).toHaveProperty('code', 'ENOENT'); + } }); - it('resolves if first fails, second is bad shasum, but third succeeds', async () => { - let reqCount = 0; - nextHandler = function sequenceHandler(req, res) { - switch (++reqCount) { - case 1: - nextHandler = sequenceHandler; - return sendErrorHandler(req, res); - case 2: - nextHandler = sequenceHandler; - return createSendHandler('bar')(req, res); - default: - return createSendHandler('foo')(req, res); - } - }; + describe('reties download retries: number of times', () => { + it('resolves if retries = 1 and first attempt fails', async () => { + handlers.push(sendErrorHandler, createSendHandler('foo')); - await download({ - log, - url: serverUrl, - destination: TMP_DESTINATION, - sha256: FOO_SHA256, - retries: 2, + await downloadToDisk({ + log, + url: serverUrl, + destination: TMP_DESTINATION, + shaChecksum: FOO_SHA256, + shaAlgorithm: 'sha256', + maxAttempts: 2, + retryDelaySecMultiplier: 0.1, + }); + + expect(readFileSync(TMP_DESTINATION, 'utf8')).toBe('foo'); + expect(logWritter.messages).toMatchInlineSnapshot(` + Array [ + " debg [1/2] Attempting download of TEST_SERVER_URL sha256", + " debg Download failed: Request failed with status code 500", + " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", + " info Retrying in 0.1 seconds", + " debg [2/2] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded TEST_SERVER_URL and verified checksum", + ] + `); }); - }); - it('makes 6 requests if `retries: 5` and all failed', async () => { - let reqCount = 0; - nextHandler = function sequenceHandler(req, res) { - reqCount += 1; - nextHandler = sequenceHandler; - sendErrorHandler(req, res); - }; + it('resolves if first fails, second is bad shasum, but third succeeds', async () => { + handlers.push(sendErrorHandler, createSendHandler('bar'), createSendHandler('foo')); - try { - await download({ + await downloadToDisk({ log, url: serverUrl, destination: TMP_DESTINATION, - sha256: FOO_SHA256, - retries: 5, + shaChecksum: FOO_SHA256, + shaAlgorithm: 'sha256', + maxAttempts: 3, + retryDelaySecMultiplier: 0.1, }); - throw new Error('Expected download() to reject'); - } catch (error) { - expect(error).toHaveProperty( - 'message', - expect.stringContaining('Request failed with status code 500') + + expect(logWritter.messages).toMatchInlineSnapshot(` + Array [ + " debg [1/3] Attempting download of TEST_SERVER_URL sha256", + " debg Download failed: Request failed with status code 500", + " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", + " info Retrying in 0.1 seconds", + " debg [2/3] Attempting download of TEST_SERVER_URL sha256", + " debg Download failed: Downloaded checksum fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9 does not match the expected sha256 checksum.", + " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", + " info Retrying in 0.2 seconds", + " debg [3/3] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded TEST_SERVER_URL and verified checksum", + ] + `); + }); + + it('makes 5 requests if `maxAttempts: 5` and all failed', async () => { + const promise = downloadToDisk({ + log, + url: serverUrl, + destination: TMP_DESTINATION, + shaChecksum: FOO_SHA256, + shaAlgorithm: 'sha256', + maxAttempts: 5, + retryDelaySecMultiplier: 0.1, + }); + await expect(promise).rejects.toMatchInlineSnapshot( + `[Error: Request failed with status code 500]` ); - expect(reqCount).toBe(6); - } + expect(logWritter.messages).toMatchInlineSnapshot(` + Array [ + " debg [1/5] Attempting download of TEST_SERVER_URL sha256", + " debg Download failed: Request failed with status code 500", + " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", + " info Retrying in 0.1 seconds", + " debg [2/5] Attempting download of TEST_SERVER_URL sha256", + " debg Download failed: Request failed with status code 500", + " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", + " info Retrying in 0.2 seconds", + " debg [3/5] Attempting download of TEST_SERVER_URL sha256", + " debg Download failed: Request failed with status code 500", + " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", + " info Retrying in 0.30000000000000004 seconds", + " debg [4/5] Attempting download of TEST_SERVER_URL sha256", + " debg Download failed: Request failed with status code 500", + " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", + " info Retrying in 0.4 seconds", + " debg [5/5] Attempting download of TEST_SERVER_URL sha256", + " debg Download failed: Request failed with status code 500", + " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", + ] + `); + }); }); -}); -describe('sha256 option not supplied', () => { - it('refuses to download', async () => { - try { + describe('sha256 option not supplied', () => { + it('refuses to download', async () => { // @ts-expect-error missing sha256 param is intentional - await download({ + const promise = downloadToDisk({ log, url: 'http://google.com', destination: TMP_DESTINATION, }); - throw new Error('expected download() to reject'); - } catch (error) { - expect(error).toHaveProperty('message', expect.stringContaining('refusing to download')); - } + await expect(promise).rejects.toMatchInlineSnapshot( + `[Error: undefined checksum of http://google.com not provided, refusing to download.]` + ); + }); + }); +}); + +describe('downloadToString', () => { + it('returns a string from the server', async () => { + handlers.push(createSendHandler('foo bar')); + + const result = await downloadToString({ log, url: serverUrl }); + expect(result).toBe('foo bar'); + expect(logWritter.messages).toMatchInlineSnapshot(` + Array [ + " debg [1/3] Attempting download to string of [TEST_SERVER_URL]", + " succ Downloaded [TEST_SERVER_URL]", + ] + `); + }); + + it(`throws when expectStatus doesn't match`, async () => { + handlers.push(createSendHandler('foo')); + + const promise = downloadToString({ + log, + url: serverUrl, + expectStatus: 201, + maxAttempts: 1, + }); + await expect(promise).rejects.toMatchInlineSnapshot( + `[Error: Request failed with status code 200]` + ); + expect(logWritter.messages).toMatchInlineSnapshot(` + Array [ + " debg [1/1] Attempting download to string of [TEST_SERVER_URL]", + " warn Download failed: Request failed with status code 200", + " debg [200/OK] response: foo", + ] + `); + }); + + it(`retries when expectStatus doesn't match`, async () => { + handlers.push( + (_, res) => { + res.statusCode = 500; + res.end('something went wrong'); + }, + (_, res) => { + res.statusCode = 404; + res.end('not found'); + }, + (_, res) => { + res.statusCode = 201; + res.end('bar'); + } + ); + + const result = await downloadToString({ + log, + url: serverUrl, + expectStatus: 201, + retryDelaySecMultiplier: 0.1, + }); + + expect(result).toBe('bar'); + expect(logWritter.messages).toMatchInlineSnapshot(` + Array [ + " debg [1/3] Attempting download to string of [TEST_SERVER_URL]", + " warn Download failed: Request failed with status code 500", + " debg [500/Internal Server Error] response: something went wrong", + " info Retrying in 0.1 seconds", + " debg [2/3] Attempting download to string of [TEST_SERVER_URL]", + " warn Download failed: Request failed with status code 404", + " debg [404/Not Found] response: not found", + " info Retrying in 0.2 seconds", + " debg [3/3] Attempting download to string of [TEST_SERVER_URL]", + " succ Downloaded [TEST_SERVER_URL]", + ] + `); }); }); diff --git a/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts index 31374d205097..f5de91794ae1 100644 --- a/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/download_node_builds_task.test.ts @@ -24,7 +24,7 @@ expect.addSnapshotSerializer(createAnyInstanceSerializer(ToolingLog)); const { getNodeDownloadInfo } = jest.requireMock('./node_download_info'); const { getNodeShasums } = jest.requireMock('./node_shasums'); -const { download } = jest.requireMock('../../lib/download'); +const { downloadToDisk } = jest.requireMock('../../lib/download'); const log = new ToolingLog(); const testWriter = new ToolingLogCollectingWriter(); @@ -55,7 +55,7 @@ async function setup({ failOnUrl }: { failOnUrl?: string } = {}) { 'win32:downloadName': 'win32:sha256', }); - download.mockImplementation(({ url }: any) => { + downloadToDisk.mockImplementation(({ url }: any) => { if (url === failOnUrl) { throw new Error('Download failed for reasons'); } @@ -69,14 +69,15 @@ it('downloads node builds for each platform', async () => { await DownloadNodeBuilds.run(config, log, []); - expect(download.mock.calls).toMatchInlineSnapshot(` + expect(downloadToDisk.mock.calls).toMatchInlineSnapshot(` Array [ Array [ Object { "destination": "linux:downloadPath", "log": , - "retries": 3, - "sha256": "linux:sha256", + "maxAttempts": 3, + "shaAlgorithm": "sha256", + "shaChecksum": "linux:sha256", "url": "linux:url", }, ], @@ -84,8 +85,9 @@ it('downloads node builds for each platform', async () => { Object { "destination": "linux:downloadPath", "log": , - "retries": 3, - "sha256": "linux:sha256", + "maxAttempts": 3, + "shaAlgorithm": "sha256", + "shaChecksum": "linux:sha256", "url": "linux:url", }, ], @@ -93,8 +95,9 @@ it('downloads node builds for each platform', async () => { Object { "destination": "darwin:downloadPath", "log": , - "retries": 3, - "sha256": "darwin:sha256", + "maxAttempts": 3, + "shaAlgorithm": "sha256", + "shaChecksum": "darwin:sha256", "url": "darwin:url", }, ], @@ -102,8 +105,9 @@ it('downloads node builds for each platform', async () => { Object { "destination": "darwin:downloadPath", "log": , - "retries": 3, - "sha256": "darwin:sha256", + "maxAttempts": 3, + "shaAlgorithm": "sha256", + "shaChecksum": "darwin:sha256", "url": "darwin:url", }, ], @@ -111,8 +115,9 @@ it('downloads node builds for each platform', async () => { Object { "destination": "win32:downloadPath", "log": , - "retries": 3, - "sha256": "win32:sha256", + "maxAttempts": 3, + "shaAlgorithm": "sha256", + "shaChecksum": "win32:sha256", "url": "win32:url", }, ], diff --git a/src/dev/build/tasks/nodejs/download_node_builds_task.ts b/src/dev/build/tasks/nodejs/download_node_builds_task.ts index c0c7a399f84c..9bc46def4696 100644 --- a/src/dev/build/tasks/nodejs/download_node_builds_task.ts +++ b/src/dev/build/tasks/nodejs/download_node_builds_task.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { download, GlobalTask } from '../../lib'; +import { downloadToDisk, GlobalTask } from '../../lib'; import { getNodeShasums } from './node_shasums'; import { getNodeDownloadInfo } from './node_download_info'; @@ -18,12 +18,13 @@ export const DownloadNodeBuilds: GlobalTask = { await Promise.all( config.getNodePlatforms().map(async (platform) => { const { url, downloadPath, downloadName } = getNodeDownloadInfo(config, platform); - await download({ + await downloadToDisk({ log, url, - sha256: shasums[downloadName], + shaChecksum: shasums[downloadName], + shaAlgorithm: 'sha256', destination: downloadPath, - retries: 3, + maxAttempts: 3, }); }) ); diff --git a/src/dev/build/tasks/nodejs/node_shasums.test.ts b/src/dev/build/tasks/nodejs/node_shasums.test.ts index 4b1482f9c082..20d1f297b9ae 100644 --- a/src/dev/build/tasks/nodejs/node_shasums.test.ts +++ b/src/dev/build/tasks/nodejs/node_shasums.test.ts @@ -47,15 +47,12 @@ c4edece2c0aa68e816c4e067f397eb12e9d0c81bb37b3d349dbaf47cf246b0b7 win-x86/node.l 6a2ee7a0b0074ece27d171418d82ce25a60b87750ec30c5c9fbeaaca8c206fa5 win-x86/node_pdb.7z 1b44176d888c1bc6a6b05fcc6234031b3b8a58da9de8b99661088f998ac5e269 win-x86/node_pdb.zip`; -jest.mock('axios', () => ({ - async get(url: string) { +jest.mock('../../lib/download', () => ({ + async downloadToString({ url }: { url: string }) { expect(url).toBe( 'https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v8.9.4/SHASUMS256.txt' ); - return { - status: 200, - data: mockResponse, - }; + return mockResponse; }, })); diff --git a/src/dev/build/tasks/nodejs/node_shasums.ts b/src/dev/build/tasks/nodejs/node_shasums.ts index 973dc86977a8..ca588352ce7b 100644 --- a/src/dev/build/tasks/nodejs/node_shasums.ts +++ b/src/dev/build/tasks/nodejs/node_shasums.ts @@ -6,29 +6,22 @@ * Side Public License, v 1. */ -import axios from 'axios'; import { ToolingLog } from '@kbn/dev-utils'; +import { downloadToString } from '../../lib/download'; export async function getNodeShasums(log: ToolingLog, nodeVersion: string) { const url = `https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v${nodeVersion}/SHASUMS256.txt`; log.debug('Downloading shasum values for node version', nodeVersion, 'from', url); - const { status, data } = await axios.get(url); + const checksum = await downloadToString({ log, url, expectStatus: 200 }); - if (status !== 200) { - throw new Error(`${url} failed with a ${status} response`); - } + return checksum.split('\n').reduce((acc: Record, line: string) => { + const [sha, platform] = line.split(' '); - return data - .toString('utf8') - .split('\n') - .reduce((acc: Record, line: string) => { - const [sha, platform] = line.split(' '); - - return { - ...acc, - [platform]: sha, - }; - }, {}); + return { + ...acc, + [platform]: sha, + }; + }, {}); } diff --git a/src/dev/build/tasks/patch_native_modules_task.ts b/src/dev/build/tasks/patch_native_modules_task.ts index 37cb72905378..be7fa5b50a07 100644 --- a/src/dev/build/tasks/patch_native_modules_task.ts +++ b/src/dev/build/tasks/patch_native_modules_task.ts @@ -10,7 +10,17 @@ import path from 'path'; import { ToolingLog } from '@kbn/dev-utils'; -import { deleteAll, download, gunzip, untar, Task, Config, Build, Platform, read } from '../lib'; +import { + deleteAll, + downloadToDisk, + gunzip, + untar, + Task, + Config, + Build, + Platform, + read, +} from '../lib'; const DOWNLOAD_DIRECTORY = '.native_modules'; @@ -100,12 +110,13 @@ async function patchModule( log.debug(`Patching ${pkg.name} binaries from ${archive.url} to ${extractPath}`); await deleteAll([extractPath], log); - await download({ + await downloadToDisk({ log, url: archive.url, destination: downloadPath, - sha256: archive.sha256, - retries: 3, + shaChecksum: archive.sha256, + shaAlgorithm: 'sha256', + maxAttempts: 3, }); switch (pkg.extractMethod) { case 'gunzip':