Add retries + improve logging for bundled package build task

elastic · Feb 17, 2022 · 3fd3aa4 · 3fd3aa4
1 parent 5eb58f7
commit 3fd3aa4
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 59 deletions.
diff --git a/src/dev/build/lib/download.ts b/src/dev/build/lib/download.ts
@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-import { openSync, writeSync, unlinkSync, closeSync } from 'fs';
+import { openSync, writeSync, unlinkSync, closeSync, statSync } from 'fs';
 import { dirname } from 'path';
 import { setTimeout } from 'timers/promises';
 
@@ -39,6 +39,7 @@ interface DownloadToDiskOptions {
   shaAlgorithm: string;
   maxAttempts?: number;
   retryDelaySecMultiplier?: number;
+  skipChecksumCheck?: boolean;
 }
 export async function downloadToDisk({
   log,
@@ -48,8 +49,9 @@ export async function downloadToDisk({
   shaAlgorithm,
   maxAttempts = 1,
   retryDelaySecMultiplier = 5,
+  skipChecksumCheck = false,
 }: DownloadToDiskOptions) {
-  if (!shaChecksum) {
+  if (!shaChecksum && !skipChecksumCheck) {
     throw new Error(`${shaAlgorithm} checksum of ${url} not provided, refusing to download.`);
   }
 
@@ -69,7 +71,7 @@ export async function downloadToDisk({
     try {
       log.debug(
         `[${attempt}/${maxAttempts}] Attempting download of ${url}`,
-        chalk.dim(shaAlgorithm)
+        skipChecksumCheck ? '' : chalk.dim(shaAlgorithm)
       );
 
       const response = await Axios.request({
@@ -83,30 +85,39 @@ export async function downloadToDisk({
       }
 
       const hash = createHash(shaAlgorithm);
+
       await new Promise((resolve, reject) => {
         response.data.on('data', (chunk: Buffer) => {
-          hash.update(chunk);
+          if (!skipChecksumCheck) {
+            hash.update(chunk);
+          }
+
           writeSync(fileHandle, chunk);
         });
 
         response.data.on('error', reject);
         response.data.on('end', resolve);
       });
 
-      const downloadedSha = hash.digest('hex');
-      if (downloadedSha !== shaChecksum) {
-        throw new Error(
-          `Downloaded checksum ${downloadedSha} does not match the expected ${shaAlgorithm} checksum.`
-        );
+      if (!skipChecksumCheck) {
+        const downloadedSha = hash.digest('hex');
+        if (downloadedSha !== shaChecksum) {
+          throw new Error(
+            `Downloaded checksum ${downloadedSha} does not match the expected ${shaAlgorithm} checksum.`
+          );
+        }
       }
     } catch (_error) {
       error = _error;
     } finally {
       closeSync(fileHandle);
+
+      const fileStats = statSync(destination);
+      log.debug(`Downloaded ${fileStats.size} bytes to ${destination}`);
     }
 
     if (!error) {
-      log.debug(`Downloaded ${url} and verified checksum`);
+      log.debug(`Downloaded ${url} ${skipChecksumCheck ? '' : 'and verified checksum'}`);
       return;
     }
 

diff --git a/src/dev/build/tasks/bundle_fleet_packages.ts b/src/dev/build/tasks/bundle_fleet_packages.ts
@@ -12,12 +12,8 @@ import JSON5 from 'json5';
 // @ts-expect-error untyped internal module used to prevent axios from using xhr adapter in tests
 import AxiosHttpAdapter from 'axios/lib/adapters/http';
 
-import { ToolingLog } from '@kbn/dev-utils';
-import { closeSync, openSync, writeSync } from 'fs';
-import { dirname } from 'path';
 import { readCliArgs } from '../args';
-
-import { Task, read, mkdirp } from '../lib';
+import { Task, read, downloadToDisk } from '../lib';
 
 const BUNDLED_PACKAGES_DIR = 'x-pack/plugins/fleet/server/bundled_packages';
 
@@ -43,15 +39,30 @@ export const BundleFleetPackages: Task = {
     const configFilePath = config.resolveFromRepo('fleet_packages.json');
     const fleetPackages = (await read(configFilePath)) || '[]';
 
+    const parsedFleetPackages: FleetPackage[] = JSON5.parse(fleetPackages);
+
+    log.debug(
+      `Found configured bundled packages: ${parsedFleetPackages
+        .map((fleetPackage) => `${fleetPackage.name}-${fleetPackage.version}`)
+        .join(', ')}`
+    );
+
     await Promise.all(
-      JSON5.parse(fleetPackages).map(async (fleetPackage: FleetPackage) => {
+      parsedFleetPackages.map(async (fleetPackage) => {
         const archivePath = `${fleetPackage.name}-${fleetPackage.version}.zip`;
         const archiveUrl = `${eprUrl}/epr/${fleetPackage.name}/${fleetPackage.name}-${fleetPackage.version}.zip`;
 
         const destination = build.resolvePath(BUNDLED_PACKAGES_DIR, archivePath);
 
         try {
-          await downloadPackageArchive({ log, url: archiveUrl, destination });
+          await downloadToDisk({
+            log,
+            url: archiveUrl,
+            destination,
+            shaChecksum: '',
+            shaAlgorithm: 'sha512',
+            skipChecksumCheck: true,
+          });
         } catch (error) {
           log.warning(`Failed to download bundled package archive ${archivePath}`);
           log.warning(error);
@@ -60,45 +71,3 @@ export const BundleFleetPackages: Task = {
     );
   },
 };
-
-/**
- * We need to skip the checksum process on Fleet's bundled packages for now because we can't reliably generate
- * a consistent checksum for the `.zip` file returned from the EPR service. This download process should be updated
- * to verify packages using the proposed package signature field provided in https://github.com/elastic/elastic-package/issues/583
- */
-async function downloadPackageArchive({
-  log,
-  url,
-  destination,
-}: {
-  log: ToolingLog;
-  url: string;
-  destination: string;
-}) {
-  log.info(`Downloading bundled package from ${url}`);
-
-  await mkdirp(dirname(destination));
-  const file = openSync(destination, 'w');
-
-  try {
-    const response = await axios.request({
-      url,
-      responseType: 'stream',
-      adapter: AxiosHttpAdapter,
-    });
-
-    await new Promise((resolve, reject) => {
-      response.data.on('data', (chunk: Buffer) => {
-        writeSync(file, chunk);
-      });
-
-      response.data.on('error', reject);
-      response.data.on('end', resolve);
-    });
-  } catch (error) {
-    log.warning(`Error downloading bundled package from ${url}`);
-    log.warning(error);
-  } finally {
-    closeSync(file);
-  }
-}