fix minio with concurrent tests

cotton-coding · Mar 20, 2024 · 315d3e7 · 315d3e7
1 parent 3c048e7
commit 315d3e7
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 38 deletions.
diff --git a/adapters/minio/package.json b/adapters/minio/package.json
@@ -31,8 +31,8 @@
 		"prepublishOnly": "npm run build && publint"
 	},
 	"devDependencies": {
-		"@loom-io/test-utils": "workspace:*",
-		"@loom-io/interface-tests": "workspace:*"
+		"@loom-io/interface-tests": "workspace:*",
+		"@loom-io/test-utils": "workspace:*"
 	},
 	"peerDependencies": {
 		"@loom-io/core": "workspace:*"

diff --git a/adapters/minio/src/core/adapter.ts b/adapters/minio/src/core/adapter.ts
@@ -2,6 +2,12 @@ import type { BucketItem, Client }	from 'minio';
 import { ObjectDirent } from './object-dirent.js';
 import { FileHandler } from './file-handler.js';
 import type { SourceAdapter, rmdirOptions, ObjectDirentInterface } from '@loom-io/core';
+import { removePrecedingSlash } from '@loom-io/common';
+
+function addTailSlash(path: string): string {
+	return path.endsWith('/') ? path : `${path}/`;
+}
+
 export class Adapter implements SourceAdapter {
 	constructor(
 		protected s3: Client,
@@ -43,7 +49,7 @@ export class Adapter implements SourceAdapter {
 	}
 
 	async dirExists(path: string): Promise<boolean> {
-		const pathWithTailSlash = path.endsWith('/') ? path : path + '/';
+		const pathWithTailSlash = addTailSlash(path);
 		if(path === '/') {
 			return true;
 		}
@@ -52,31 +58,30 @@ export class Adapter implements SourceAdapter {
 
 
 	async mkdir(path: string): Promise<void> {
-		const pathWithTailSlash = path.endsWith('/') ? path : `${path}/`;
-		if(pathWithTailSlash === '/') {
+		const pathWithTailSlash = removePrecedingSlash(addTailSlash(path));
+		if(pathWithTailSlash === '') {
 			return;
 		}
 		await this.s3.putObject(this.bucket, pathWithTailSlash, Buffer.alloc(0));
 	}
 
 	protected async rmdirRecursive(bucket: string, path: string): Promise<void> {
-		const objects = await this.s3.listObjectsV2(bucket, path);
+		path = removePrecedingSlash(addTailSlash(path));
+		const objects = await this.s3.listObjectsV2(bucket, path, true);
 		for await (const obj of objects) {
 			await this.s3.removeObject(bucket, obj.name);
 		}
 	}
 
 	protected async rmdirForce(path: string): Promise<void> {
-		const pathWithTailSlash = path.endsWith('/') ? path : `${path}/`;
+		const pathWithTailSlash = removePrecedingSlash(addTailSlash(path));
+		if(pathWithTailSlash === '') {
+			await this.rmdirRecursive(this.bucket, pathWithTailSlash);
+			return;
+		}
 		await this.s3.removeObject(this.bucket, pathWithTailSlash, { forceDelete: true });
 	}
 
-	protected async rmDirRecursive(bucket: string, path: string): Promise<void> {
-		const objects = await this.filesInDir(path);
-		const names = objects.map((obj) => obj.name).filter((name): name is string => typeof name === 'string');
-		await this.s3.removeObjects(bucket, names);
-	}
-
 	protected async dirHasFiles(path: string): Promise<boolean> {
 		const pathWithTailSlash = path.endsWith('/') ? path : `${path}/`;
 		const bucketStream = await this.s3.listObjectsV2(this.bucket, pathWithTailSlash);
@@ -115,8 +120,8 @@ export class Adapter implements SourceAdapter {
 	}
 
 	async readdir(path: string): Promise<ObjectDirentInterface[]> {
-		const pathWithTailSlash = path.endsWith('/') ? path : `${path}/`;
-		const bucketStream = await this.s3.listObjectsV2(this.bucket, pathWithTailSlash === '/' ? '' : pathWithTailSlash);
+		const pathWithTailSlash = removePrecedingSlash(addTailSlash(path));
+		const bucketStream = await this.s3.listObjectsV2(this.bucket, pathWithTailSlash, false);
 		return new Promise((resolve, reject) => {
 			const pathObjects: ObjectDirent[] = [];
 			bucketStream.on('data', (data) => {

diff --git a/adapters/minio/test/adapter.spec.ts b/adapters/minio/test/adapter.spec.ts
@@ -2,7 +2,8 @@ import { TestAdapter } from '@loom-io/interface-tests';
 import { Adapter } from '../src/core/adapter';
 import { Client } from 'minio';
 
-const DEFAULT_BUCKET: string = `cotton-coding-${Math.random().toString(36).substring(7)}`;
+const DEFAULT_BUCKET: string = 'cotton-coding';
+
 
 const s3Client = new Client({
 	endPoint: 'play.min.io',
@@ -12,39 +13,32 @@ const s3Client = new Client({
 	secretKey: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG',
 });
 
+
 async function createBucketIfNotExists(client: Client, bucket: string) {
 	const doesBucketExist = await client.bucketExists(bucket);
 	if (!doesBucketExist) {
 		await client.makeBucket(bucket);
 	}
 }
 
-async function cleanAndRemoveBucket(client: Client, bucket: string) {
-	const bucketStream = await client.listObjects(bucket);
-	return new Promise((resolve, reject) => {
-		bucketStream.on('data', (data) => {
-			client.removeObject(bucket, data.name || data.prefix || '', { forceDelete: true });
-		});
-		bucketStream.on('end', () => {
-			resolve(false);
-		});
-		bucketStream.on('error', (err) => {
-			reject(err);
-		});
-	});
+
+async function cleanBucketForce(client: Client, bucket: string) {
+	const objects = await client.listObjectsV2(bucket, '', true);
+	for await (const obj of objects) {
+		await client.removeObject(bucket, obj.name);
+	}
 }
 
 
+
 async function createAdapter(client, bucket): Promise<Adapter> {
 	await createBucketIfNotExists(client, bucket);
+	await cleanBucketForce(client, bucket);
 	return new Adapter(
 		client,
 		bucket
 	);
 }
 
 
-TestAdapter(await createAdapter(s3Client, DEFAULT_BUCKET), {
-	beforeEach: createBucketIfNotExists.bind(null, s3Client, DEFAULT_BUCKET),
-	afterEach: cleanAndRemoveBucket.bind(null, s3Client, DEFAULT_BUCKET)
-});
+TestAdapter(await createAdapter(s3Client, DEFAULT_BUCKET));
diff --git a/tests/interface-tests/src/adapter.ts b/tests/interface-tests/src/adapter.ts
@@ -87,8 +87,6 @@ export const TestAdapter = (adapter: SourceAdapter, config?: TestAdapterOptions
 
 	describe.concurrent('Adapter', async () => {
 
-		let path: string;
-
 		if (config?.beforeAll) {
 			beforeAll(config.beforeAll);
 		}
@@ -98,7 +96,6 @@ export const TestAdapter = (adapter: SourceAdapter, config?: TestAdapterOptions
 		}
 
 		beforeEach(async () => {
-			path = getRandomPath();
 			if(config?.beforeEach) {
 				await config.beforeEach();
 			}
@@ -115,7 +112,7 @@ export const TestAdapter = (adapter: SourceAdapter, config?: TestAdapterOptions
 		});
 
 		test('rmdir', async () => {
-			const path = getRandomPath('test/long/mkdir');
+			const path = getRandomPath('test/other/mkdir');
 			const [subPath] = splitTailingPath(path);
 			await adapter.mkdir(subPath);
 			await adapter.mkdir(path);
@@ -125,6 +122,7 @@ export const TestAdapter = (adapter: SourceAdapter, config?: TestAdapterOptions
 		});
 
 		test('rmdir with file should fail', async () => {
+			const path = getRandomPath('test/other/mkdir');
 			const fileName = 'file.txt';
 			await adapter.mkdir(path);
 			await adapter.writeFile(`${path}/${fileName}`, 'test');
@@ -139,6 +137,7 @@ export const TestAdapter = (adapter: SourceAdapter, config?: TestAdapterOptions
 		});
 
 		test('exists with path', async () => {
+			const path = getRandomPath();
 			await adapter.mkdir(path);
 			const subPath = path.split('/').slice(0,-1).join('/');
 			expect( await adapter.dirExists(path)).toBe(true);
@@ -334,7 +333,7 @@ export const TestAdapter = (adapter: SourceAdapter, config?: TestAdapterOptions
 				const read = await adapter.readdir('/');
 				expect(read.length).toBe(amount);
 				finish();
-			});
+			}, 25000);
 
 
 			test.sequential('should handle root slash', async () => {