From 148e5d2a11237c33ee4f2d01e31a62ac588b191f Mon Sep 17 00:00:00 2001 From: busma13 Date: Thu, 7 Nov 2024 15:01:43 -0700 Subject: [PATCH 1/2] add skip docker image deletion option to test-runner --- packages/scripts/src/cmds/test.ts | 8 ++++++++ packages/scripts/src/helpers/config.ts | 1 + packages/scripts/src/helpers/k8s-env/index.ts | 3 ++- packages/scripts/src/helpers/kind.ts | 6 ++++-- packages/scripts/src/helpers/scripts.ts | 14 +++++++++++--- packages/scripts/src/helpers/test-runner/index.ts | 13 +++++++------ .../scripts/src/helpers/test-runner/interfaces.ts | 1 + .../scripts/src/helpers/test-runner/services.ts | 14 +++++++++++--- packages/scripts/test/service-spec.ts | 3 ++- packages/scripts/test/test-runner-spec.ts | 3 ++- 10 files changed, 49 insertions(+), 17 deletions(-) diff --git a/packages/scripts/src/cmds/test.ts b/packages/scripts/src/cmds/test.ts index 227ef5305d9..bb22d88d965 100644 --- a/packages/scripts/src/cmds/test.ts +++ b/packages/scripts/src/cmds/test.ts @@ -22,6 +22,7 @@ type Options = { packages?: PackageInfo[]; 'ignore-mount': boolean; 'test-platform': string; + 'skip-image-deletion': boolean; }; const jestArgs = getExtraArgs(); @@ -101,6 +102,11 @@ const cmd: CommandModule = { default: config.TEST_PLATFORM, choices: ['native', 'kubernetes', 'kubernetesV2'] }) + .option('skip-image-deletion', { + description: 'Skip the deletion of docker images from cache after loading into docker.\n This is useful if a CI job calls `ts-scripts test` more than once.', + type: 'boolean', + default: config.SKIP_IMAGE_DELETION, + }) .positional('packages', { description: 'Runs the tests for one or more package and/or an asset, if none specified it will run all of the tests', coerce(arg) { @@ -126,6 +132,7 @@ const cmd: CommandModule = { const ignoreMount = hoistJestArg(argv, 'ignore-mount', 'boolean'); const testPlatform = hoistJestArg(argv, 'test-platform', 'string') as 'native' | 'kubernetes' | 'kubernetesV2'; const kindClusterName = testPlatform === 'native' ? 'default' : 'k8s-e2e'; + const skipImageDeletion = hoistJestArg(argv, 'skip-image-deletion', 'boolean'); if (debug && watch) { throw new Error('--debug and --watch conflict, please set one or the other'); @@ -147,6 +154,7 @@ const cmd: CommandModule = { ignoreMount, testPlatform, kindClusterName, + skipImageDeletion }); }, }; diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index c4cebedf9f7..a1250e87413 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -206,3 +206,4 @@ export const { export const DOCKER_IMAGES_PATH = './images'; export const DOCKER_IMAGE_LIST_PATH = `${DOCKER_IMAGES_PATH}/image-list.txt`; export const DOCKER_CACHE_PATH = '/tmp/docker_cache'; +export const SKIP_IMAGE_DELETION = toBoolean(process.env.SKIP_IMAGE_DELETION) || false; diff --git a/packages/scripts/src/helpers/k8s-env/index.ts b/packages/scripts/src/helpers/k8s-env/index.ts index af5c95fdbe6..d36b34e8e8e 100644 --- a/packages/scripts/src/helpers/k8s-env/index.ts +++ b/packages/scripts/src/helpers/k8s-env/index.ts @@ -106,7 +106,8 @@ export async function launchK8sEnv(options: K8sEnvOptions) { reportCoverage: false, useExistingServices: false, ignoreMount: false, - testPlatform: options.clusteringType + testPlatform: options.clusteringType, + skipImageDeletion: false }); try { diff --git a/packages/scripts/src/helpers/kind.ts b/packages/scripts/src/helpers/kind.ts index 6f4721b3430..ee790a0fa7e 100644 --- a/packages/scripts/src/helpers/kind.ts +++ b/packages/scripts/src/helpers/kind.ts @@ -80,7 +80,7 @@ export class Kind { // TODO: check that image is loaded before we continue async loadServiceImage( - serviceName: string, serviceImage: string, version: string + serviceName: string, serviceImage: string, version: string, skipDelete: boolean ): Promise { let subprocess; try { @@ -95,7 +95,9 @@ export class Kind { subprocess = await execaCommand(`gunzip -d ${filePath}`); signale.info(`${subprocess.command}: successful`); subprocess = await execaCommand(`kind load --name ${this.clusterName} image-archive ${tarPath}`); - fs.rmSync(tarPath); + if (!skipDelete) { + fs.rmSync(tarPath); + } } else { subprocess = await execaCommand(`kind load --name ${this.clusterName} docker-image ${serviceImage}:${version}`); } diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index fcbe3289496..56dc123de73 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -456,11 +456,15 @@ async function dockerImageRm(image: string): Promise { /** * Unzips and loads a Docker image from a Docker cache - * If successful the image will be deleted from the cache + * If successful and skipDelete is false the image will be deleted from the cache * @param {string} imageName Name of the image to load + * @param {boolean} skipDelete Skip removal of docker image from cache * @returns {Promise} Whether or not the image loaded successfully */ -export async function loadThenDeleteImageFromCache(imageName: string): Promise { +export async function loadThenDeleteImageFromCache( + imageName: string, + skipDelete = false +): Promise { signale.time(`unzip and load ${imageName}`); const fileName = imageName.trim().replace(/[/:]/g, '_'); const filePath = path.join(config.DOCKER_CACHE_PATH, `${fileName}.tar.gz`); @@ -478,7 +482,11 @@ export async function loadThenDeleteImageFromCache(imageName: string): Promise>> = { } }; -export async function loadOrPullServiceImages(suite: string): Promise { +export async function loadOrPullServiceImages( + suite: string, + skipImageDeletion: boolean +): Promise { const launchServices = getServicesForSuite(suite); try { @@ -209,7 +212,7 @@ export async function loadOrPullServiceImages(suite: string): Promise { if (fs.existsSync(config.DOCKER_CACHE_PATH)) { await Promise.all(images.map(async (imageName) => { - const success = await loadThenDeleteImageFromCache(imageName); + const success = await loadThenDeleteImageFromCache(imageName, skipImageDeletion); if (!success) { loadFailedList.push(imageName); } @@ -834,7 +837,12 @@ async function startService(options: TestOptions, service: Service): Promise<() if (options.testPlatform === 'kubernetes' || options.testPlatform === 'kubernetesV2') { const kind = new Kind(config.K8S_VERSION, options.kindClusterName); - await kind.loadServiceImage(service, services[service].image, version); + await kind.loadServiceImage( + service, + services[service].image, + version, + options.skipImageDeletion + ); await k8sStopService(service); await k8sStartService(service, services[service].image, version, kind); return () => { }; diff --git a/packages/scripts/test/service-spec.ts b/packages/scripts/test/service-spec.ts index a715db0e988..e6469d1b7a0 100644 --- a/packages/scripts/test/service-spec.ts +++ b/packages/scripts/test/service-spec.ts @@ -110,7 +110,8 @@ describe('services', () => { encryptMinio: false, ignoreMount: false, testPlatform: 'native', - kindClusterName: 'default' + kindClusterName: 'default', + skipImageDeletion: false }; let services: any; diff --git a/packages/scripts/test/test-runner-spec.ts b/packages/scripts/test/test-runner-spec.ts index 514be3a02fb..a3cb8516b62 100644 --- a/packages/scripts/test/test-runner-spec.ts +++ b/packages/scripts/test/test-runner-spec.ts @@ -21,7 +21,8 @@ describe('Test Runner Helpers', () => { encryptMinio: false, ignoreMount: true, testPlatform: 'native', - kindClusterName: 'default' + kindClusterName: 'default', + skipImageDeletion: false }; function makeTestOptions(input: Partial): TestOptions { From 2c44300129a6d24b0982995a4452df88dd92d628 Mon Sep 17 00:00:00 2001 From: busma13 Date: Fri, 8 Nov 2024 13:02:47 -0700 Subject: [PATCH 2/2] bump: (minor) @terascope/scripts@1.5.0 --- packages/scripts/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scripts/package.json b/packages/scripts/package.json index 8f3ee0624b8..20d79fcf8bc 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/scripts", "displayName": "Scripts", - "version": "1.4.2", + "version": "1.5.0", "description": "A collection of terascope monorepo scripts", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme", "bugs": {