diff --git a/src/dev/build/lib/download.ts b/src/dev/build/lib/download.ts index c67407095b37e..cb569e8e738ae 100644 --- a/src/dev/build/lib/download.ts +++ b/src/dev/build/lib/download.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { openSync, writeSync, unlinkSync, closeSync } from 'fs'; +import { openSync, writeSync, unlinkSync, closeSync, statSync } from 'fs'; import { dirname } from 'path'; import { setTimeout } from 'timers/promises'; @@ -39,6 +39,7 @@ interface DownloadToDiskOptions { shaAlgorithm: string; maxAttempts?: number; retryDelaySecMultiplier?: number; + skipChecksumCheck?: boolean; } export async function downloadToDisk({ log, @@ -48,8 +49,9 @@ export async function downloadToDisk({ shaAlgorithm, maxAttempts = 1, retryDelaySecMultiplier = 5, + skipChecksumCheck = false, }: DownloadToDiskOptions) { - if (!shaChecksum) { + if (!shaChecksum && !skipChecksumCheck) { throw new Error(`${shaAlgorithm} checksum of ${url} not provided, refusing to download.`); } @@ -69,7 +71,7 @@ export async function downloadToDisk({ try { log.debug( `[${attempt}/${maxAttempts}] Attempting download of ${url}`, - chalk.dim(shaAlgorithm) + skipChecksumCheck ? '' : chalk.dim(shaAlgorithm) ); const response = await Axios.request({ @@ -83,30 +85,47 @@ export async function downloadToDisk({ } const hash = createHash(shaAlgorithm); - await new Promise((resolve, reject) => { + let bytesWritten = 0; + + await new Promise((resolve, reject) => { response.data.on('data', (chunk: Buffer) => { - hash.update(chunk); - writeSync(fileHandle, chunk); + if (!skipChecksumCheck) { + hash.update(chunk); + } + + const bytes = writeSync(fileHandle, chunk); + bytesWritten += bytes; }); response.data.on('error', reject); - response.data.on('end', resolve); + response.data.on('end', () => { + if (bytesWritten === 0) { + return reject(new Error(`No bytes written when downloading ${url}`)); + } + + return resolve(); + }); }); - const downloadedSha = hash.digest('hex'); - if (downloadedSha !== shaChecksum) { - throw new Error( - `Downloaded checksum ${downloadedSha} does not match the expected ${shaAlgorithm} checksum.` - ); + if (!skipChecksumCheck) { + const downloadedSha = hash.digest('hex'); + if (downloadedSha !== shaChecksum) { + throw new Error( + `Downloaded checksum ${downloadedSha} does not match the expected ${shaAlgorithm} checksum.` + ); + } } } catch (_error) { error = _error; } finally { closeSync(fileHandle); + + const fileStats = statSync(destination); + log.debug(`Downloaded ${fileStats.size} bytes to ${destination}`); } if (!error) { - log.debug(`Downloaded ${url} and verified checksum`); + log.debug(`Downloaded ${url} ${skipChecksumCheck ? '' : 'and verified checksum'}`); return; } diff --git a/src/dev/build/lib/integration_tests/download.test.ts b/src/dev/build/lib/integration_tests/download.test.ts index 7f885fdc02d28..26e374fa102db 100644 --- a/src/dev/build/lib/integration_tests/download.test.ts +++ b/src/dev/build/lib/integration_tests/download.test.ts @@ -146,10 +146,12 @@ describe('downloadToDisk', () => { expect(logWritter.messages).toMatchInlineSnapshot(` Array [ " debg [1/2] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded 0 bytes to TMP_DIR/__tmp_download_js_test_file__", " debg Download failed: Request failed with status code 500", " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", " info Retrying in 0.1 seconds", " debg [2/2] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded 3 bytes to TMP_DIR/__tmp_download_js_test_file__", " debg Downloaded TEST_SERVER_URL and verified checksum", ] `); @@ -171,14 +173,17 @@ describe('downloadToDisk', () => { expect(logWritter.messages).toMatchInlineSnapshot(` Array [ " debg [1/3] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded 0 bytes to TMP_DIR/__tmp_download_js_test_file__", " debg Download failed: Request failed with status code 500", " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", " info Retrying in 0.1 seconds", " debg [2/3] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded 3 bytes to TMP_DIR/__tmp_download_js_test_file__", " debg Download failed: Downloaded checksum fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9 does not match the expected sha256 checksum.", " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", " info Retrying in 0.2 seconds", " debg [3/3] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded 3 bytes to TMP_DIR/__tmp_download_js_test_file__", " debg Downloaded TEST_SERVER_URL and verified checksum", ] `); @@ -200,22 +205,27 @@ describe('downloadToDisk', () => { expect(logWritter.messages).toMatchInlineSnapshot(` Array [ " debg [1/5] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded 0 bytes to TMP_DIR/__tmp_download_js_test_file__", " debg Download failed: Request failed with status code 500", " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", " info Retrying in 0.1 seconds", " debg [2/5] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded 0 bytes to TMP_DIR/__tmp_download_js_test_file__", " debg Download failed: Request failed with status code 500", " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", " info Retrying in 0.2 seconds", " debg [3/5] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded 0 bytes to TMP_DIR/__tmp_download_js_test_file__", " debg Download failed: Request failed with status code 500", " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", " info Retrying in 0.30000000000000004 seconds", " debg [4/5] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded 0 bytes to TMP_DIR/__tmp_download_js_test_file__", " debg Download failed: Request failed with status code 500", " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", " info Retrying in 0.4 seconds", " debg [5/5] Attempting download of TEST_SERVER_URL sha256", + " debg Downloaded 0 bytes to TMP_DIR/__tmp_download_js_test_file__", " debg Download failed: Request failed with status code 500", " debg Deleting downloaded data at TMP_DIR/__tmp_download_js_test_file__", ] diff --git a/src/dev/build/tasks/bundle_fleet_packages.ts b/src/dev/build/tasks/bundle_fleet_packages.ts index 0fd584354494f..cda4f47d1210e 100644 --- a/src/dev/build/tasks/bundle_fleet_packages.ts +++ b/src/dev/build/tasks/bundle_fleet_packages.ts @@ -6,18 +6,10 @@ * Side Public License, v 1. */ -import axios from 'axios'; import JSON5 from 'json5'; -// @ts-expect-error untyped internal module used to prevent axios from using xhr adapter in tests -import AxiosHttpAdapter from 'axios/lib/adapters/http'; - -import { ToolingLog } from '@kbn/dev-utils'; -import { closeSync, openSync, writeSync } from 'fs'; -import { dirname } from 'path'; import { readCliArgs } from '../args'; - -import { Task, read, mkdirp } from '../lib'; +import { Task, read, downloadToDisk } from '../lib'; const BUNDLED_PACKAGES_DIR = 'x-pack/plugins/fleet/server/bundled_packages'; @@ -43,15 +35,31 @@ export const BundleFleetPackages: Task = { const configFilePath = config.resolveFromRepo('fleet_packages.json'); const fleetPackages = (await read(configFilePath)) || '[]'; + const parsedFleetPackages: FleetPackage[] = JSON5.parse(fleetPackages); + + log.debug( + `Found configured bundled packages: ${parsedFleetPackages + .map((fleetPackage) => `${fleetPackage.name}-${fleetPackage.version}`) + .join(', ')}` + ); + await Promise.all( - JSON5.parse(fleetPackages).map(async (fleetPackage: FleetPackage) => { + parsedFleetPackages.map(async (fleetPackage) => { const archivePath = `${fleetPackage.name}-${fleetPackage.version}.zip`; const archiveUrl = `${eprUrl}/epr/${fleetPackage.name}/${fleetPackage.name}-${fleetPackage.version}.zip`; const destination = build.resolvePath(BUNDLED_PACKAGES_DIR, archivePath); try { - await downloadPackageArchive({ log, url: archiveUrl, destination }); + await downloadToDisk({ + log, + url: archiveUrl, + destination, + shaChecksum: '', + shaAlgorithm: 'sha512', + skipChecksumCheck: true, + maxAttempts: 3, + }); } catch (error) { log.warning(`Failed to download bundled package archive ${archivePath}`); log.warning(error); @@ -60,45 +68,3 @@ export const BundleFleetPackages: Task = { ); }, }; - -/** - * We need to skip the checksum process on Fleet's bundled packages for now because we can't reliably generate - * a consistent checksum for the `.zip` file returned from the EPR service. This download process should be updated - * to verify packages using the proposed package signature field provided in https://github.com/elastic/elastic-package/issues/583 - */ -async function downloadPackageArchive({ - log, - url, - destination, -}: { - log: ToolingLog; - url: string; - destination: string; -}) { - log.info(`Downloading bundled package from ${url}`); - - await mkdirp(dirname(destination)); - const file = openSync(destination, 'w'); - - try { - const response = await axios.request({ - url, - responseType: 'stream', - adapter: AxiosHttpAdapter, - }); - - await new Promise((resolve, reject) => { - response.data.on('data', (chunk: Buffer) => { - writeSync(file, chunk); - }); - - response.data.on('error', reject); - response.data.on('end', resolve); - }); - } catch (error) { - log.warning(`Error downloading bundled package from ${url}`); - log.warning(error); - } finally { - closeSync(file); - } -}