Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add test for macos and windows #45

Merged
merged 9 commits into from
Nov 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,12 @@ on:
jobs:
test:
name: lint & test
runs-on: ubuntu-latest

strategy:
matrix:
node-version: [18, 20, 22, lts/*]
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- name: pnpm-setup
Expand All @@ -35,6 +37,7 @@ jobs:
- name: Test
run: pnpm test:ci
- name: Upload coverage reports to Codecov
if: runner.os != 'windows'
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
Expand Down
17 changes: 9 additions & 8 deletions adapters/in-memory/src/core/adapter.spec.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { normalize, sep } from 'node:path';
import { MemoryDirectory, MEMORY_TYPE, MemoryRoot, MemoryObject, MemoryFile } from '../definitions.js';
import { AlreadyExistsException, NotFoundException } from '../exceptions.js';
import { Adapter } from './adapter.js';
Expand Down Expand Up @@ -157,7 +158,7 @@ describe('Memory Adapter internal functions', async () => {
const path = 'test/some/cotton/coding/file.txt';
const file = adapter.createObject(path, MEMORY_TYPE.FILE);
expect(adapter._storage.content.length).toBe(1);
const parts = path.split('/');
const parts = path.split(sep);
const fileName = parts.pop();
let ref: MemoryDirectory | MemoryRoot = adapter._storage;
for(const part of parts) {
Expand Down Expand Up @@ -208,37 +209,37 @@ describe('Memory Adapter internal functions', async () => {
adapter.setTestStorage();
const dir = adapter.getLastPartOfPath('test', MEMORY_TYPE.DIRECTORY);
expect(dir).toBe(adapter._storage.content[0]);
const file = adapter.getLastPartOfPath('test/file.txt', MEMORY_TYPE.FILE);
const file = adapter.getLastPartOfPath(normalize('test/file.txt'), MEMORY_TYPE.FILE);
expect(file).toBe(adapter._storage.content[0].content[0]);
});

test('get last part of path (directory)', () => {
adapter.setTestStorage();
const dir = adapter.getLastPartOfPath('cotton-coding', MEMORY_TYPE.DIRECTORY);
expect(dir).toBe(adapter._storage.content[1]);
const file = adapter.getLastPartOfPath('cotton-coding/sub-dir', MEMORY_TYPE.DIRECTORY);
const file = adapter.getLastPartOfPath(normalize('cotton-coding/sub-dir'), MEMORY_TYPE.DIRECTORY);
expect(file).toBe(adapter._storage.content[1].content[0]);
});

test('get last part of path (find empty dir)', () => {
adapter.setTestStorage();
const dir = adapter.getLastPartOfPath('cotton-coding/empty-dir/', MEMORY_TYPE.DIRECTORY);
const dir = adapter.getLastPartOfPath(normalize('cotton-coding/empty-dir/'), MEMORY_TYPE.DIRECTORY);
expect(dir).toBe(adapter._storage.content[1].content[1]);
});

test('get last part of path (not found with type)', () => {
adapter.setTestStorage();
expect(() => adapter.getLastPartOfPath('cotton-coding/sub-dir', MEMORY_TYPE.FILE)).toThrow(NotFoundException);
expect(() => adapter.getLastPartOfPath(normalize('cotton-coding/sub-dir'), MEMORY_TYPE.FILE)).toThrow(NotFoundException);
});

test('get last part of path (not found)', () => {
adapter.setTestStorage();
expect(() => adapter.getLastPartOfPath('/cotton-coding/sub-dir/file.txt')).toThrow(NotFoundException);
expect(() => adapter.getLastPartOfPath(normalize('/cotton-coding/sub-dir/file.txt'))).toThrow(NotFoundException);
});


test('Crate object by filename with deep path and existing directories', () => {
const path = 'test/some/cotton/coding/file.txt';
test('Create object by filename with deep path and existing directories', () => {
const path = normalize('test/some/cotton/coding/file.txt');
adapter.createObject(path, MEMORY_TYPE.FILE);
expect(() => adapter.createObject(path, MEMORY_TYPE.FILE)).toThrow(AlreadyExistsException);
expect(adapter._storage.content.length).toBe(1);
Expand Down
13 changes: 6 additions & 7 deletions adapters/in-memory/src/core/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { ObjectDirent } from './object-dirent.js';
import { MEMORY_TYPE, MemoryDirectory, MemoryFile, MemoryObject, MemoryRoot } from '../definitions.js';
import { isMemoryDirectoryAndMatchNamePrepared } from '../utils/validations.js';
import { removePrecedingAndTrailingSlash, splitTailingPath } from '@loom-io/common';
import { basename, dirname } from 'path';
import { basename, dirname, sep } from 'node:path';
export class Adapter implements SourceAdapter {

protected storage: MemoryRoot;
Expand Down Expand Up @@ -33,10 +33,10 @@ export class Adapter implements SourceAdapter {
protected getLastPartOfPath(path: string | undefined, ref?: MEMORY_TYPE): MemoryObject | MemoryRoot;
protected getLastPartOfPath(path: string | undefined, ref?: MEMORY_TYPE): MemoryObject | MemoryRoot {
path = path?.trim();
if(path === undefined || path === '' || path === '/' || path === '.') {
if(path === undefined || path === '' || path === sep || path === '.') {
return this.storage;
}
const parts = removePrecedingAndTrailingSlash(path).split('/');
const parts = removePrecedingAndTrailingSlash(path).split(sep);
const lastPart = parts.pop();
if(lastPart === undefined) {
return this.storage;
Expand Down Expand Up @@ -96,8 +96,7 @@ export class Adapter implements SourceAdapter {
protected createObject(path: string, ref: MEMORY_TYPE.FILE): MemoryFile;
protected createObject(path: string, ref: MEMORY_TYPE.DIRECTORY): MemoryDirectory;
protected createObject(path: string, ref: MEMORY_TYPE): MemoryObject {

const parts = removePrecedingAndTrailingSlash(path).split('/');
const parts = removePrecedingAndTrailingSlash(path).split(sep);
try {
const lastPart = this.getLastPartOfPath(path);
throw new AlreadyExistsException(path, lastPart);
Expand Down Expand Up @@ -138,7 +137,7 @@ export class Adapter implements SourceAdapter {

mkdir(path: string): void {
path = path.trim();
if(path === '/' || path === '') {
if(path === sep || path === '') {
return;
}
try {
Expand Down Expand Up @@ -214,7 +213,7 @@ export class Adapter implements SourceAdapter {
file.mtime = new Date();
} catch (err) {
if(err instanceof NotFoundException) {
if(removePrecedingAndTrailingSlash(path).split('/').length === err.depth + 1) {
if(removePrecedingAndTrailingSlash(path).split(sep).length === err.depth + 1) {
const [,tail] = splitTailingPath(path);
if(tail === undefined) {
throw new Error('Invalid path'); // TODO: Create a custom exception
Expand Down
85 changes: 47 additions & 38 deletions adapters/minio/src/core/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,10 @@
DirectoryNotEmptyException,
PathNotFoundException,
} from "@loom-io/core";
import { removePrecedingSlash } from "@loom-io/common";
import { join } from "path";
import { removePrecedingSlash, addTailingSlash } from "@loom-io/common";
import { join, normalize } from "node:path";
import { S3Error } from "minio/dist/esm/errors.mjs";

function addTailSlash(path: string): string {
return path.endsWith("/") ? path : `${path}/`;
}

type AdapterStat = {
size: number;
Expand All @@ -28,7 +25,7 @@
protected bucket: string,
) {}
async deleteFile(path: string): Promise<void> {
await this.s3.removeObject(this.bucket, path);
await this.s3.removeObject(this.bucket, this.translatePath(path));
}

get raw() {
Expand All @@ -39,7 +36,17 @@
return this.bucket;
}

protected translatePath(path: string) {
if(path.match(/[A-Za-z]{1}:/)) {
return path.slice(2).replaceAll('\\', '/');

Check warning on line 41 in adapters/minio/src/core/adapter.ts

View check run for this annotation

Codecov / codecov/patch

adapters/minio/src/core/adapter.ts#L41

Added line #L41 was not covered by tests
} else if (path.includes('\\')) {
return path.replaceAll('\\', '/');
}

Check warning on line 44 in adapters/minio/src/core/adapter.ts

View check run for this annotation

Codecov / codecov/patch

adapters/minio/src/core/adapter.ts#L43-L44

Added lines #L43 - L44 were not covered by tests
return path;
}

protected async exists(path: string): Promise<boolean> {

const bucketStream = this.s3.listObjectsV2(this.bucket, path);
return new Promise((resolve, reject) => {
bucketStream.on("data", () => {
Expand All @@ -56,35 +63,35 @@
}

async fileExists(path: string): Promise<boolean> {
return this.exists(path);
return this.exists(this.translatePath(path));
}

async dirExists(path: string): Promise<boolean> {
const pathWithTailSlash = addTailSlash(path);
if (path === "/") {
const pathWithTailSlash = this.translatePath(addTailingSlash(path));
if (pathWithTailSlash === "/") {
return true;
}
return this.exists(pathWithTailSlash);
}

async mkdir(path: string): Promise<void> {
const pathWithTailSlash = removePrecedingSlash(addTailSlash(path));
const pathWithTailSlash = this.translatePath(removePrecedingSlash(addTailingSlash(path)));
if (pathWithTailSlash === "") {
return;
}
await this.s3.putObject(this.bucket, pathWithTailSlash, Buffer.alloc(0));
}

protected async rmdirRecursive(bucket: string, path: string): Promise<void> {
path = removePrecedingSlash(addTailSlash(path));
const objects = await this.s3.listObjectsV2(bucket, path, true);
const pathWithTailSlash = this.translatePath(removePrecedingSlash(addTailingSlash(path)));
const objects = await this.s3.listObjectsV2(bucket, pathWithTailSlash, true);
for await (const obj of objects) {
await this.s3.removeObject(bucket, obj.name);
}
}

protected async rmdirForce(path: string): Promise<void> {
const pathWithTailSlash = removePrecedingSlash(addTailSlash(path));
const pathWithTailSlash = this.translatePath(removePrecedingSlash(addTailingSlash(path)));

Check warning on line 94 in adapters/minio/src/core/adapter.ts

View check run for this annotation

Codecov / codecov/patch

adapters/minio/src/core/adapter.ts#L94

Added line #L94 was not covered by tests
if (pathWithTailSlash === "") {
await this.rmdirRecursive(this.bucket, pathWithTailSlash);
return;
Expand All @@ -95,6 +102,7 @@
}

protected async dirHasFiles(path: string): Promise<boolean> {
path = this.translatePath(path);
const pathWithTailSlash = path.endsWith("/") ? path : `${path}/`;
const bucketStream = await this.s3.listObjectsV2(
this.bucket,
Expand All @@ -117,8 +125,25 @@
});
}

async rmdir(path: string, options: rmdirOptions = {}): Promise<void> {
if (options.force) {
await this.rmdirForce(path);
return;

Check warning on line 131 in adapters/minio/src/core/adapter.ts

View check run for this annotation

Codecov / codecov/patch

adapters/minio/src/core/adapter.ts#L130-L131

Added lines #L130 - L131 were not covered by tests
} else if (options.recursive) {
await this.rmdirRecursive(this.bucket, path);
return;
} else {
path = this.translatePath(path);
if (await this.dirHasFiles(path)) {
throw new DirectoryNotEmptyException(path);
}
const pathWithTailSlash = path.endsWith("/") ? path : `${path}/`;
await this.s3.removeObject(this.bucket, pathWithTailSlash);
}
}

async readdir(path: string): Promise<ObjectDirentInterface[]> {
const pathWithTailSlash = removePrecedingSlash(addTailSlash(path));
const pathWithTailSlash = this.translatePath(removePrecedingSlash(addTailingSlash(path)));
const bucketStream = await this.s3.listObjectsV2(
this.bucket,
pathWithTailSlash,
Expand All @@ -144,25 +169,9 @@
});
});
}

async rmdir(path: string, options: rmdirOptions = {}): Promise<void> {
if (options.force) {
await this.rmdirForce(path);
return;
} else if (options.recursive) {
await this.rmdirRecursive(this.bucket, path);
return;
} else {
if (await this.dirHasFiles(path)) {
throw new DirectoryNotEmptyException(path);
}
const pathWithTailSlash = path.endsWith("/") ? path : `${path}/`;
await this.s3.removeObject(this.bucket, pathWithTailSlash);
}
}


async stat(path: string) {
const stat = await this.s3.statObject(this.bucket, path);
const stat = await this.s3.statObject(this.bucket, this.translatePath(path));
return {
size: stat.size,
mtime: stat.lastModified,
Expand All @@ -175,7 +184,7 @@
path: string,
encoding?: BufferEncoding,
): Promise<Buffer | string> {
const stream = await this.s3.getObject(this.bucket, path);
const stream = await this.s3.getObject(this.bucket, this.translatePath(path));
return new Promise((resolve, reject) => {
const buffers: Buffer[] = [];
stream.on("data", (data) => {
Expand All @@ -195,14 +204,14 @@

async writeFile(path: string, data: Buffer | string): Promise<void> {
const buffer = Buffer.from(data);
await this.s3.putObject(this.bucket, path, buffer, buffer.length, {
await this.s3.putObject(this.bucket, this.translatePath(path), buffer, buffer.length, {
"Content-Type": "text/plain",
});
return;
}

async openFile(path: string): Promise<FileHandler> {
return new FileHandler(this, this.bucket, path);
return new FileHandler(this, this.bucket, this.translatePath(path));
}

async isCopyable(adapter: SourceAdapter): Promise<boolean> {
Expand All @@ -217,15 +226,15 @@
const conds = new CopyConditions();
await this.s3.copyObject(
this.bucket,
dest,
join(this.bucket, src),
this.translatePath(dest),
this.translatePath(join(this.bucket, src)),
conds,
);
} catch (err) {
if (err instanceof S3Error) {
if (err.code === "NoSuchKey") {
// @ts-expect-error property code does not exist on S3Error
throw new PathNotFoundException(err.key);
throw new PathNotFoundException(normalize(err.key));
}

throw err;
Expand Down
3 changes: 2 additions & 1 deletion adapters/minio/src/core/object-dirent.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { ObjectDirentInterface } from '@loom-io/core';
import { BucketItem } from 'minio';
import { addPrecedingAndTailingSlash } from '@loom-io/common';
import { normalize } from 'node:path';

export class ObjectDirent implements ObjectDirentInterface{

Expand Down Expand Up @@ -44,7 +45,7 @@ export class ObjectDirent implements ObjectDirentInterface{

get path() {
const [path] = this.getPathAndName();
return addPrecedingAndTailingSlash(path);
return normalize(addPrecedingAndTailingSlash(path));
}

}
13 changes: 8 additions & 5 deletions adapters/node-fs/src/core/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import type { SourceAdapter, rmdirOptions, ObjectDirentInterface } from '@loom-io/core';
import { DirectoryNotEmptyException, PathNotFoundException } from '@loom-io/core';
import { PathLike } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
import { dirname, join, normalize, resolve, sep } from 'node:path';
import { isNodeErrnoExpression } from '../utils/error-handling.js';
import { ObjectDirent } from './object-dirent.js';
export class Adapter implements SourceAdapter {
Expand All @@ -14,19 +14,22 @@
rootdir: PathLike = process.cwd(),
) {
const fullPath = resolve(rootdir.toString());
this.rootdir = fullPath.endsWith('/') ? fullPath : `${fullPath}/`;
this.rootdir = fullPath.endsWith(sep) ? fullPath : normalize(`${fullPath}/`);
}

get raw() {
return fs;
}

protected getFullPath(path: string): string {
if(path.match(/^[A-Za-z]{1}:/) && !['', '\\'].includes(this.rootdir)) {
return join(this.rootdir, path.slice(2));
}

Check warning on line 27 in adapters/node-fs/src/core/adapter.ts

View check run for this annotation

Codecov / codecov/patch

adapters/node-fs/src/core/adapter.ts#L26-L27

Added lines #L26 - L27 were not covered by tests
return join(this.rootdir || '', path);
}

protected getRelativePath(path: string): string {
return path.replace(this.rootdir, '/');
return path.replace(this.rootdir, sep);

Check warning on line 32 in adapters/node-fs/src/core/adapter.ts

View check run for this annotation

Codecov / codecov/patch

adapters/node-fs/src/core/adapter.ts#L32

Added line #L32 was not covered by tests
}

protected async exists(path: string, ref: number): Promise<boolean> {
Expand Down Expand Up @@ -65,11 +68,11 @@
const fullPath = this.getFullPath(path);
if(options.recursive || options.force) {
await fs.rm(fullPath, options);
if(path === '/' || path === '') {
if(path === sep || path === '') {
await fs.mkdir(this.rootdir);
}
} else {
if(path !== '/' && path !== '') {
if(path !== sep && path !== '') {
await fs.rmdir(fullPath);
}
}
Expand Down
2 changes: 1 addition & 1 deletion adapters/node-fs/src/core/object-dirent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export class ObjectDirent implements ObjectDirentInterface{
}

get path() {
const pathFromRelativeRoot = this._dirent.path.slice(this._rootPath.length);
const pathFromRelativeRoot = this._dirent.parentPath.slice(this._rootPath.length);
if(process.version.startsWith('v18') && pathFromRelativeRoot.endsWith(this.name)) {
return addPrecedingAndTailingSlash(pathFromRelativeRoot.slice(0, -this.name.length));
}
Expand Down
Loading