diff --git a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageOperationsTestSuite.java b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageOperationsTestSuite.java index c6e16ba1c..3e3cf0078 100644 --- a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageOperationsTestSuite.java +++ b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageOperationsTestSuite.java @@ -63,8 +63,14 @@ public abstract class StorageOperationsTestSuite { protected IDataContext dataContext; // Windows limits individual path segments to 255 chars - private static final String LONG_PATH_TXT_FILE = "long_" + "A".repeat(246) + ".txt"; - private static final String LONG_PATH_DIR = "long_" + "A".repeat(250); + + private static String makeLongPath(String prefix, String suffix) { + return prefix + "A".repeat(255 - prefix.length() - suffix.length()) + suffix; + } + + private static String makeLongDirPath(String prefix) { + return prefix + "A".repeat(255 - prefix.length()); + } // ----------------------------------------------------------------------------------------------------------------- @@ -74,11 +80,11 @@ public abstract class StorageOperationsTestSuite { @Test void testExists_file() throws Exception { - var prepare = makeSmallFile("test_file.txt", storage, dataContext); + var prepare = makeSmallFile("testExists_file.txt", storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var filePresent = storage.exists("test_file.txt", dataContext); - var fileNotPresent = storage.exists("other_file.txt", dataContext); + var filePresent = storage.exists("testExists_file.txt", dataContext); + var fileNotPresent = storage.exists("testExists_file_other.txt", dataContext); waitFor(TEST_TIMEOUT, filePresent, fileNotPresent); @@ -89,26 +95,26 @@ void testExists_file() throws Exception { @Test void testExists_longPath() throws Exception { - var prepare = makeSmallFile(LONG_PATH_TXT_FILE, storage, dataContext); + var longPath = makeLongPath("testExists_longPath", ".txt"); + + var prepare = makeSmallFile(longPath, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var filePresent = storage.exists(LONG_PATH_TXT_FILE, dataContext); - var fileNotPresent = storage.exists("other_file.txt", dataContext); + var filePresent = storage.exists(longPath, dataContext); - waitFor(TEST_TIMEOUT, filePresent, fileNotPresent); + waitFor(TEST_TIMEOUT, filePresent); Assertions.assertTrue(getResultOf(filePresent)); - Assertions.assertFalse(getResultOf(fileNotPresent)); } @Test void testExists_emptyFile() throws Exception { var empty = dataContext.arrowAllocator().getEmpty(); - var prepare = makeFile("test_file.txt", empty, storage, dataContext); + var prepare = makeFile("testExists_emptyFile.txt", empty, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var emptyFileExist = storage.exists("test_file.txt", dataContext); + var emptyFileExist = storage.exists("testExists_emptyFile.txt", dataContext); waitFor(TEST_TIMEOUT, emptyFileExist); @@ -119,11 +125,11 @@ void testExists_emptyFile() throws Exception { @Test void testExists_dir() throws Exception { - var prepare = storage.mkdir("test_dir", false, dataContext); + var prepare = storage.mkdir("testExists_dir", false, dataContext); waitFor(TEST_TIMEOUT, prepare); - var dirPresent = storage.exists("test_dir", dataContext); - var dirNotPresent = storage.exists("other_dir", dataContext); + var dirPresent = storage.exists("testExists_dir", dataContext); + var dirNotPresent = storage.exists("testExists_dir_other", dataContext); waitFor(TEST_TIMEOUT, dirPresent, dirNotPresent); @@ -134,11 +140,11 @@ void testExists_dir() throws Exception { @Test void testExists_parentDir() throws Exception { - var prepare = storage.mkdir("parent_dir/child_dir", true, dataContext); + var prepare = storage.mkdir("testExists_parentDir/child_dir", true, dataContext); waitFor(TEST_TIMEOUT, prepare); - var dirPresent = storage.exists("parent_dir", dataContext); - var dirNotPresent = storage.exists("other_dir", dataContext); + var dirPresent = storage.exists("testExists_parentDir", dataContext); + var dirNotPresent = storage.exists("testExists_parentDir_other", dataContext); waitFor(TEST_TIMEOUT, dirPresent, dirNotPresent); @@ -176,10 +182,10 @@ void testSize_ok() throws Exception { var content = Bytes.copyToBuffer(bytes, dataContext.arrowAllocator()); var expectedSize = content.readableBytes(); - var prepare = makeFile("test_file.txt", content, storage, dataContext); + var prepare = makeFile("testSize_ok.txt", content, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var size = storage.size("test_file.txt", dataContext); + var size = storage.size("testSize_ok.txt", dataContext); waitFor(TEST_TIMEOUT, size); @@ -189,14 +195,16 @@ void testSize_ok() throws Exception { @Test void testSize_longPath() throws Exception { + var longPath = makeLongPath("testSize_longPath", ".txt"); + var bytes = "Content of a certain size\n".getBytes(StandardCharsets.UTF_8); var content = Bytes.copyToBuffer(bytes, dataContext.arrowAllocator()); var expectedSize = content.readableBytes(); - var prepare = makeFile(LONG_PATH_TXT_FILE, content, storage, dataContext); + var prepare = makeFile(longPath, content, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var size = storage.size(LONG_PATH_TXT_FILE, dataContext); + var size = storage.size(longPath, dataContext); waitFor(TEST_TIMEOUT, size); @@ -207,10 +215,10 @@ void testSize_longPath() throws Exception { void testSize_emptyFile() throws Exception { var empty = dataContext.arrowAllocator().getEmpty(); - var prepare = makeFile("test_file.txt", empty, storage, dataContext); + var prepare = makeFile("testSize_emptyFile.txt", empty, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var size = storage.size("test_file.txt", dataContext); + var size = storage.size("testSize_emptyFile.txt", dataContext); waitFor(TEST_TIMEOUT, size); @@ -220,10 +228,10 @@ void testSize_emptyFile() throws Exception { @Test void testSize_dir() { - var prepare = storage.mkdir("test_dir", false, dataContext); + var prepare = storage.mkdir("testSize_dir", false, dataContext); waitFor(TEST_TIMEOUT, prepare); - var size = storage.size("test_dir", dataContext); + var size = storage.size("testSize_dir", dataContext); waitFor(TEST_TIMEOUT, size); @@ -233,7 +241,7 @@ void testSize_dir() { @Test void testSize_missing() { - var size = storage.size("missing_file.txt", dataContext); + var size = storage.size("testSize_missing.txt", dataContext); waitFor(TEST_TIMEOUT, size); @@ -274,16 +282,16 @@ void testStat_fileOk() throws Exception { var expectedSize = content.readableBytes(); var prepare = storage .mkdir("some_dir", false, dataContext) - .thenCompose(x -> makeFile("some_dir/test_file.txt", content, storage, dataContext)); + .thenCompose(x -> makeFile("testStat_fileOk/test_file.txt", content, storage, dataContext)); waitFor(TEST_TIMEOUT, prepare); - var stat = storage.stat("some_dir/test_file.txt", dataContext); + var stat = storage.stat("testStat_fileOk/test_file.txt", dataContext); waitFor(TEST_TIMEOUT, stat); var statResult = getResultOf(stat); - Assertions.assertEquals("some_dir/test_file.txt", statResult.storagePath); + Assertions.assertEquals("testStat_fileOk/test_file.txt", statResult.storagePath); Assertions.assertEquals("test_file.txt", statResult.fileName); Assertions.assertEquals(FileType.FILE, statResult.fileType); Assertions.assertEquals(expectedSize, statResult.size); @@ -292,6 +300,8 @@ void testStat_fileOk() throws Exception { @Test void testStat_fileLongPath() throws Exception { + var longPath = makeLongPath("testStat_fileLongPath", ".txt"); + // Simple case - stat a file var bytes = "Sample content for stat call\n".getBytes(StandardCharsets.UTF_8); @@ -299,18 +309,18 @@ void testStat_fileLongPath() throws Exception { var expectedSize = content.readableBytes(); var prepare = storage - .mkdir("some_dir", false, dataContext) - .thenCompose(x -> makeFile("some_dir/" + LONG_PATH_TXT_FILE, content, storage, dataContext)); + .mkdir("testStat_fileLongPath", false, dataContext) + .thenCompose(x -> makeFile("testStat_fileLongPath/" + longPath, content, storage, dataContext)); waitFor(TEST_TIMEOUT, prepare); - var stat = storage.stat("some_dir/" + LONG_PATH_TXT_FILE, dataContext); + var stat = storage.stat("testStat_fileLongPath/" + longPath, dataContext); waitFor(TEST_TIMEOUT, stat); var statResult = getResultOf(stat); - Assertions.assertEquals("some_dir/" + LONG_PATH_TXT_FILE, statResult.storagePath); - Assertions.assertEquals(LONG_PATH_TXT_FILE, statResult.fileName); + Assertions.assertEquals("testStat_fileLongPath/" + longPath, statResult.storagePath); + Assertions.assertEquals(longPath, statResult.fileName); Assertions.assertEquals(FileType.FILE, statResult.fileType); Assertions.assertEquals(expectedSize, statResult.size); } @@ -324,10 +334,10 @@ void testStat_fileMTime() throws Exception { var testStart = Instant.now(); Thread.sleep(1000); // Let time elapse before/after the test calls - var prepare = makeSmallFile("test_file.txt", storage, dataContext); + var prepare = makeSmallFile("testStat_fileMTime.txt", storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var stat = storage.stat("test_file.txt", dataContext); + var stat = storage.stat("testStat_fileMTime.txt", dataContext); waitFor(TEST_TIMEOUT, stat); Thread.sleep(1000); // Let time elapse before/after the test calls @@ -357,20 +367,20 @@ void testStat_fileATime() throws Exception { // On FAT32, atime is limited to an access date, i.e. one-day resolution - var prepare = makeSmallFile("test_file.txt", storage, dataContext); + var prepare = makeSmallFile("testStat_fileATime.txt", storage, dataContext); waitFor(TEST_TIMEOUT, prepare); var testStart = Instant.now(); Thread.sleep(10); // Let time elapse before/after the test calls var readData = - storage.size("test_file.txt", dataContext).thenCompose(size -> - storage.readChunk("test_file.txt", 0, (int)(long) size, dataContext)); + storage.size("testStat_fileATime.txt", dataContext).thenCompose(size -> + storage.readChunk("testStat_fileATime.txt", 0, (int)(long) size, dataContext)); waitFor(TEST_TIMEOUT, readData); readData.toCompletableFuture().get().close(); - var stat = storage.stat("test_file.txt", dataContext); + var stat = storage.stat("testStat_fileATime.txt", dataContext); waitFor(TEST_TIMEOUT, stat); Thread.sleep(10); // Let time elapse before/after the test calls @@ -385,15 +395,15 @@ void testStat_fileATime() throws Exception { @Test void testStat_dirOk() throws Exception { - var prepare = storage.mkdir("some_dir/test_dir", true, dataContext); + var prepare = storage.mkdir("testStat_dirOk/test_dir", true, dataContext); waitFor(TEST_TIMEOUT, prepare); - var stat = storage.stat("some_dir/test_dir", dataContext); + var stat = storage.stat("testStat_dirOk/test_dir", dataContext); waitFor(TEST_TIMEOUT, stat); var statResult = getResultOf(stat); - Assertions.assertEquals("some_dir/test_dir", statResult.storagePath); + Assertions.assertEquals("testStat_dirOk/test_dir", statResult.storagePath); Assertions.assertEquals("test_dir", statResult.fileName); Assertions.assertEquals(FileType.DIRECTORY, statResult.fileType); @@ -404,16 +414,18 @@ void testStat_dirOk() throws Exception { @Test void testStat_dirLongPath() throws Exception { - var prepare = storage.mkdir("some_dir/" + LONG_PATH_DIR, true, dataContext); + var longDir = makeLongDirPath("testStat_dirLongPath"); + + var prepare = storage.mkdir("testStat_dirLongPath/" + longDir, true, dataContext); waitFor(TEST_TIMEOUT, prepare); - var stat = storage.stat("some_dir/" + LONG_PATH_DIR, dataContext); + var stat = storage.stat("testStat_dirLongPath/" + longDir, dataContext); waitFor(TEST_TIMEOUT, stat); var statResult = getResultOf(stat); - Assertions.assertEquals("some_dir/" + LONG_PATH_DIR, statResult.storagePath); - Assertions.assertEquals(LONG_PATH_DIR, statResult.fileName); + Assertions.assertEquals("testStat_dirLongPath/" + longDir, statResult.storagePath); + Assertions.assertEquals(longDir, statResult.fileName); Assertions.assertEquals(FileType.DIRECTORY, statResult.fileType); // Size field for directories should always be set to 0 @@ -423,16 +435,16 @@ void testStat_dirLongPath() throws Exception { @Test void testStat_dirImplicitOk() throws Exception { - var prepare = storage.mkdir("some_dir/test_dir", true, dataContext); + var prepare = storage.mkdir("testStat_dirImplicitOk/test_dir", true, dataContext); waitFor(TEST_TIMEOUT, prepare); - var stat = storage.stat("some_dir", dataContext); + var stat = storage.stat("testStat_dirImplicitOk", dataContext); waitFor(TEST_TIMEOUT, stat); var statResult = getResultOf(stat); - Assertions.assertEquals("some_dir", statResult.storagePath); - Assertions.assertEquals("some_dir", statResult.fileName); + Assertions.assertEquals("testStat_dirImplicitOk", statResult.storagePath); + Assertions.assertEquals("testStat_dirImplicitOk", statResult.fileName); Assertions.assertEquals(FileType.DIRECTORY, statResult.fileType); // Size field for directories should always be set to 0 @@ -445,7 +457,7 @@ void testStat_dirMTime() throws Exception { // mtime and atime for dirs is unlikely to be supported in cloud storage buckets // So, all of these fields are optional in stat responses for directories - var prepare1 = storage.mkdir("some_dir/test_dir", true, dataContext); + var prepare1 = storage.mkdir("testStat_dirMTime/test_dir", true, dataContext); waitFor(TEST_TIMEOUT, prepare1); // "Modify" the directory by adding a file to it @@ -453,10 +465,10 @@ void testStat_dirMTime() throws Exception { var testStart = Instant.now(); Thread.sleep(10); // Let time elapse before/after the test calls - var prepare2 = makeSmallFile("some_dir/test_dir/a_file.txt", storage, dataContext); + var prepare2 = makeSmallFile("testStat_dirMTime/test_dir/a_file.txt", storage, dataContext); waitFor(TEST_TIMEOUT, prepare2); - var stat = storage.stat("some_dir/test_dir", dataContext); + var stat = storage.stat("testStat_dirMTime/test_dir", dataContext); waitFor(TEST_TIMEOUT, stat); Thread.sleep(10); // Let time elapse before/after the test calls @@ -478,8 +490,8 @@ void testStat_dirATime() throws Exception { // So, all of these fields are optional in stat responses for directories var prepare1 = storage - .mkdir("some_dir/test_dir", true, dataContext) - .thenCompose(x -> makeSmallFile("some_dir/test_dir/a_file.txt", storage, dataContext)); + .mkdir("testStat_dirATime/test_dir", true, dataContext) + .thenCompose(x -> makeSmallFile("testStat_dirATime/test_dir/a_file.txt", storage, dataContext)); waitFor(TEST_TIMEOUT, prepare1); // Access the directory by running "ls" on it @@ -487,10 +499,10 @@ void testStat_dirATime() throws Exception { var testStart = Instant.now(); Thread.sleep(10); // Let time elapse before/after the test calls - var prepare2 = storage.ls("some_dir/test_dir", dataContext); + var prepare2 = storage.ls("testStat_dirATime/test_dir", dataContext); waitFor(TEST_TIMEOUT, prepare2); - var stat = storage.stat("some_dir/test_dir", dataContext); + var stat = storage.stat("testStat_dirATime/test_dir", dataContext); waitFor(TEST_TIMEOUT, stat); Thread.sleep(10); // Let time elapse before/after the test calls @@ -521,7 +533,7 @@ void testStat_storageRoot() throws Exception { @Test void testStat_missing() { - var stat = storage.stat("does_not_exist.dat", dataContext); + var stat = storage.stat("testStat_missing.dat", dataContext); waitFor(TEST_TIMEOUT, stat); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(stat)); @@ -543,12 +555,12 @@ void testLs_ok() throws Exception { // Simple listing, dir containing one file and one sub dir - var prepare = storage.mkdir("test_dir", false, dataContext) - .thenCompose(x -> storage.mkdir("test_dir/child_1", false, dataContext)) - .thenCompose(x -> makeSmallFile("test_dir/child_2.txt", storage, dataContext)); + var prepare = storage.mkdir("testLs_ok", false, dataContext) + .thenCompose(x -> storage.mkdir("testLs_ok/child_1", false, dataContext)) + .thenCompose(x -> makeSmallFile("testLs_ok/child_2.txt", storage, dataContext)); waitFor(TEST_TIMEOUT, prepare); - var ls = storage.ls("test_dir", dataContext); + var ls = storage.ls("testLs_ok", dataContext); waitFor(TEST_TIMEOUT, ls); var listing = getResultOf(ls); @@ -559,25 +571,27 @@ void testLs_ok() throws Exception { var child2 = listing.stream().filter(e -> e.fileName.equals("child_2.txt")).findFirst(); Assertions.assertTrue(child1.isPresent()); - Assertions.assertEquals("test_dir/child_1", child1.get().storagePath); + Assertions.assertEquals("testLs_ok/child_1", child1.get().storagePath); Assertions.assertEquals(FileType.DIRECTORY, child1.get().fileType); Assertions.assertTrue(child2.isPresent()); - Assertions.assertEquals("test_dir/child_2.txt", child2.get().storagePath); + Assertions.assertEquals("testLs_ok/child_2.txt", child2.get().storagePath); Assertions.assertEquals(FileType.FILE, child2.get().fileType); } @Test void testLs_longPath() throws Exception { + var longPath = makeLongDirPath("testLs_longPath"); + // Simple listing, dir containing one file and one sub dir - var prepare = storage.mkdir(LONG_PATH_DIR, false, dataContext) - .thenCompose(x -> storage.mkdir(LONG_PATH_DIR + "/child_1", false, dataContext)) - .thenCompose(x -> makeSmallFile(LONG_PATH_DIR + "/child_2.txt", storage, dataContext)); + var prepare = storage.mkdir(longPath, false, dataContext) + .thenCompose(x -> storage.mkdir(longPath + "/child_1", false, dataContext)) + .thenCompose(x -> makeSmallFile(longPath + "/child_2.txt", storage, dataContext)); waitFor(TEST_TIMEOUT, prepare); - var ls = storage.ls(LONG_PATH_DIR, dataContext); + var ls = storage.ls(longPath, dataContext); waitFor(TEST_TIMEOUT, ls); var listing = getResultOf(ls); @@ -588,11 +602,11 @@ void testLs_longPath() throws Exception { var child2 = listing.stream().filter(e -> e.fileName.equals("child_2.txt")).findFirst(); Assertions.assertTrue(child1.isPresent()); - Assertions.assertEquals(LONG_PATH_DIR + "/child_1", child1.get().storagePath); + Assertions.assertEquals(longPath + "/child_1", child1.get().storagePath); Assertions.assertEquals(FileType.DIRECTORY, child1.get().fileType); Assertions.assertTrue(child2.isPresent()); - Assertions.assertEquals(LONG_PATH_DIR + "/child_2.txt", child2.get().storagePath); + Assertions.assertEquals(longPath + "/child_2.txt", child2.get().storagePath); Assertions.assertEquals(FileType.FILE, child2.get().fileType); } @@ -602,12 +616,12 @@ void testLs_extensions() throws Exception { // Corner case - dir with an extension, file without extension - var prepare = storage.mkdir("ls_extensions", false, dataContext) - .thenCompose(x -> storage.mkdir("ls_extensions/child_1.dat", false, dataContext)) - .thenCompose(x -> makeSmallFile("ls_extensions/child_2_file", storage, dataContext)); + var prepare = storage.mkdir("testLs_extensions", false, dataContext) + .thenCompose(x -> storage.mkdir("testLs_extensions/child_1.dat", false, dataContext)) + .thenCompose(x -> makeSmallFile("testLs_extensions/child_2_file", storage, dataContext)); waitFor(TEST_TIMEOUT, prepare); - var ls = storage.ls("ls_extensions", dataContext); + var ls = storage.ls("testLs_extensions", dataContext); waitFor(TEST_TIMEOUT, ls); var listing = getResultOf(ls); @@ -618,11 +632,11 @@ void testLs_extensions() throws Exception { var child2 = listing.stream().filter(e -> e.fileName.equals("child_2_file")).findFirst(); Assertions.assertTrue(child1.isPresent()); - Assertions.assertEquals("ls_extensions/child_1.dat", child1.get().storagePath); + Assertions.assertEquals("testLs_extensions/child_1.dat", child1.get().storagePath); Assertions.assertEquals(FileType.DIRECTORY, child1.get().fileType); Assertions.assertTrue(child2.isPresent()); - Assertions.assertEquals("ls_extensions/child_2_file", child2.get().storagePath); + Assertions.assertEquals("testLs_extensions/child_2_file", child2.get().storagePath); Assertions.assertEquals(FileType.FILE, child2.get().fileType); } @@ -631,12 +645,12 @@ void testLs_trailingSlash() throws Exception { // Storage path should be accepted with or without trailing slash - var prepare = storage.mkdir("ls_trailing_slash", false, dataContext) - .thenCompose(x -> makeSmallFile("ls_trailing_slash/some_file.txt", storage, dataContext)); + var prepare = storage.mkdir("testLs_trailingSlash", false, dataContext) + .thenCompose(x -> makeSmallFile("testLs_trailingSlash/some_file.txt", storage, dataContext)); waitFor(TEST_TIMEOUT, prepare); - var ls1 = storage.ls("ls_trailing_slash", dataContext); - var ls2 = storage.ls("ls_trailing_slash/", dataContext); + var ls1 = storage.ls("testLs_trailingSlash", dataContext); + var ls2 = storage.ls("testLs_trailingSlash/", dataContext); waitFor(TEST_TIMEOUT, ls1, ls2); var listing1 = getResultOf(ls1); @@ -651,8 +665,8 @@ void testLs_storageRootAllowed() throws Exception { // Ls is one operation that is allowed on the storage root! - var prepare = storage.mkdir("test_dir", false, dataContext) - .thenCompose(x -> makeSmallFile("test_file.txt", storage, dataContext)); + var prepare = storage.mkdir("testLs_storageRootAllowed", false, dataContext) + .thenCompose(x -> makeSmallFile("testLs_storageRootAllowed_file.txt", storage, dataContext)); waitFor(TEST_TIMEOUT, prepare); var ls = storage.ls(".", dataContext); @@ -662,15 +676,15 @@ void testLs_storageRootAllowed() throws Exception { Assertions.assertTrue(listing.size() >= 2); - var child1 = listing.stream().filter(e -> e.fileName.equals("test_dir")).findFirst(); - var child2 = listing.stream().filter(e -> e.fileName.equals("test_file.txt")).findFirst(); + var child1 = listing.stream().filter(e -> e.fileName.equals("testLs_storageRootAllowed")).findFirst(); + var child2 = listing.stream().filter(e -> e.fileName.equals("testLs_storageRootAllowed_file.txt")).findFirst(); Assertions.assertTrue(child1.isPresent()); - Assertions.assertEquals("test_dir", child1.get().storagePath); + Assertions.assertEquals("testLs_storageRootAllowed", child1.get().storagePath); Assertions.assertEquals(FileType.DIRECTORY, child1.get().fileType); Assertions.assertTrue(child2.isPresent()); - Assertions.assertEquals("test_file.txt", child2.get().storagePath); + Assertions.assertEquals("testLs_storageRootAllowed_file.txt", child2.get().storagePath); Assertions.assertEquals(FileType.FILE, child2.get().fileType); } @@ -679,10 +693,10 @@ void testLs_file() throws Exception { // Attempt to call ls on a file is an error - var prepare = makeSmallFile("test_file", storage, dataContext); + var prepare = makeSmallFile("testLs_file", storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var ls = storage.ls("test_file", dataContext); + var ls = storage.ls("testLs_file", dataContext); waitFor(TEST_TIMEOUT, ls); var fileLs = getResultOf(ls); @@ -692,8 +706,8 @@ void testLs_file() throws Exception { var stat = fileLs.get(0); Assertions.assertEquals(FileType.FILE, stat.fileType); - Assertions.assertEquals("test_file", stat.fileName); - Assertions.assertEquals("test_file", stat.storagePath); + Assertions.assertEquals("testLs_file", stat.fileName); + Assertions.assertEquals("testLs_file", stat.storagePath); } @Test @@ -701,7 +715,7 @@ void testLs_missing() { // Ls on a missing path is an error condition - var ls = storage.ls("dir_does_not_exist/", dataContext); + var ls = storage.ls("testLs_missing/", dataContext); waitFor(TEST_TIMEOUT, ls); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(ls)); @@ -723,20 +737,20 @@ void testMkdir_ok() throws Exception { // Simplest case - create a single directory - var mkdir = storage.mkdir("test_dir", false, dataContext); + var mkdir = storage.mkdir("testMkdir_ok", false, dataContext); waitFor(TEST_TIMEOUT, mkdir); Assertions.assertDoesNotThrow(() -> getResultOf(mkdir)); // Creating a single child dir when the parent already exists - var childDir = storage.mkdir("test_dir/child", false, dataContext); + var childDir = storage.mkdir("testMkdir_ok/child", false, dataContext); waitFor(TEST_TIMEOUT, childDir); Assertions.assertDoesNotThrow(() -> getResultOf(childDir)); - var dirExists = storage.exists("test_dir", dataContext); - var childExists = storage.exists("test_dir/child", dataContext); + var dirExists = storage.exists("testMkdir_ok", dataContext); + var childExists = storage.exists("testMkdir_ok/child", dataContext); waitFor(TEST_TIMEOUT, dirExists, childExists); Assertions.assertTrue(getResultOf(dirExists)); @@ -746,22 +760,24 @@ void testMkdir_ok() throws Exception { @Test void testMkdir_longPath() throws Exception { + var longPath = makeLongDirPath("testMkdir_longPath"); + // Simplest case - create a single directory - var mkdir = storage.mkdir(LONG_PATH_DIR, false, dataContext); + var mkdir = storage.mkdir(longPath, false, dataContext); waitFor(TEST_TIMEOUT, mkdir); Assertions.assertDoesNotThrow(() -> getResultOf(mkdir)); // Creating a single child dir when the parent already exists - var childDir = storage.mkdir(LONG_PATH_DIR + "/child", false, dataContext); + var childDir = storage.mkdir(longPath + "/child", false, dataContext); waitFor(TEST_TIMEOUT, childDir); Assertions.assertDoesNotThrow(() -> getResultOf(childDir)); - var dirExists = storage.exists(LONG_PATH_DIR, dataContext); - var childExists = storage.exists(LONG_PATH_DIR + "/child", dataContext); + var dirExists = storage.exists(longPath, dataContext); + var childExists = storage.exists(longPath + "/child", dataContext); waitFor(TEST_TIMEOUT, dirExists, childExists); Assertions.assertTrue(getResultOf(dirExists)); @@ -773,19 +789,19 @@ void testMkdir_dirExists() throws Exception { // It is not an error to call mkdir on an existing directory - var prepare = storage.mkdir("test_dir", false, dataContext); + var prepare = storage.mkdir("testMkdir_dirExists", false, dataContext); waitFor(TEST_TIMEOUT, prepare); - var exists1 = storage.exists("test_dir", dataContext); + var exists1 = storage.exists("testMkdir_dirExists", dataContext); waitFor(TEST_TIMEOUT, exists1); var exists1Result = getResultOf(exists1); Assertions.assertTrue(exists1Result); - var mkdir = storage.mkdir("test_dir", false, dataContext); + var mkdir = storage.mkdir("testMkdir_dirExists", false, dataContext); waitFor(TEST_TIMEOUT, mkdir); - var exists2 = storage.exists("test_dir", dataContext); + var exists2 = storage.exists("testMkdir_dirExists", dataContext); waitFor(TEST_TIMEOUT, exists2); var exists2Result = getResultOf(exists2); @@ -797,10 +813,10 @@ void testMkdir_fileExists() { // mkdir should always fail if requested dir already exists and is a file - var prepare = makeSmallFile("test_dir", storage, dataContext); + var prepare = makeSmallFile("testMkdir_fileExists", storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var mkdir = storage.mkdir("test_dir", false, dataContext); + var mkdir = storage.mkdir("testMkdir_fileExists", false, dataContext); waitFor(TEST_TIMEOUT, mkdir); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(mkdir)); @@ -812,13 +828,13 @@ void testMkdir_missingParent() throws Exception { // With recursive = false, mkdir with a missing parent should fail // Neither parent nor child dir should be created - var childDir = storage.mkdir("test_dir/child", false, dataContext); + var childDir = storage.mkdir("testMkdir_missingParent/child", false, dataContext); waitFor(TEST_TIMEOUT, childDir); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(childDir)); - var dirExists = storage.exists("test_dir", dataContext); - var childExists = storage.exists("test_dir/child", dataContext); + var dirExists = storage.exists("testMkdir_missingParent", dataContext); + var childExists = storage.exists("testMkdir_missingParent/child", dataContext); waitFor(TEST_TIMEOUT, dirExists, childExists); Assertions.assertFalse(getResultOf(dirExists)); @@ -830,13 +846,13 @@ void testMkdir_recursiveOk() throws Exception { // mkdir, recursive = true, create parent and child dir in a single call - var mkdir = storage.mkdir("test_dir/child", true, dataContext); + var mkdir = storage.mkdir("testMkdir_recursiveOk/child", true, dataContext); waitFor(TEST_TIMEOUT, mkdir); Assertions.assertDoesNotThrow(() -> getResultOf(mkdir)); - var dirExists = storage.exists("test_dir", dataContext); - var childExists = storage.exists("test_dir/child", dataContext); + var dirExists = storage.exists("testMkdir_recursiveOk", dataContext); + var childExists = storage.exists("testMkdir_recursiveOk/child", dataContext); waitFor(TEST_TIMEOUT, dirExists, childExists); Assertions.assertTrue(getResultOf(dirExists)); @@ -848,16 +864,16 @@ void testMkdir_recursiveDirExists() throws Exception { // mkdir, when recursive = true it is not an error if the target dir already exists - var prepare = storage.mkdir("test_dir/child", true, dataContext); + var prepare = storage.mkdir("testMkdir_recursiveDirExists/child", true, dataContext); waitFor(TEST_TIMEOUT, prepare); - var mkdir = storage.mkdir("test_dir/child", true, dataContext); + var mkdir = storage.mkdir("testMkdir_recursiveDirExists/child", true, dataContext); waitFor(TEST_TIMEOUT, mkdir); Assertions.assertDoesNotThrow(() -> getResultOf(mkdir)); - var dirExists = storage.exists("test_dir", dataContext); - var childExists = storage.exists("test_dir/child", dataContext); + var dirExists = storage.exists("testMkdir_recursiveDirExists", dataContext); + var childExists = storage.exists("testMkdir_recursiveDirExists/child", dataContext); waitFor(TEST_TIMEOUT, dirExists, childExists); Assertions.assertTrue(getResultOf(dirExists)); @@ -870,12 +886,12 @@ void testMkdir_recursiveFileExists() { // mkdir should always fail if requested dir already exists and is a file var prepare = storage - .mkdir("test_dir", false, dataContext) - .thenCompose(x -> makeSmallFile("test_dir/child", storage, dataContext)); + .mkdir("testMkdir_recursiveFileExists", false, dataContext) + .thenCompose(x -> makeSmallFile("testMkdir_recursiveFileExists/child", storage, dataContext)); waitFor(TEST_TIMEOUT, prepare); - var mkdir = storage.mkdir("test_dir/child", false, dataContext); + var mkdir = storage.mkdir("testMkdir_recursiveFileExists/child", false, dataContext); waitFor(TEST_TIMEOUT, mkdir); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(mkdir)); @@ -898,13 +914,13 @@ void testMkdir_storageRoot() { @Test void testMkdir_unicode() throws Exception { - var mkdir = storage.mkdir("你好/你好", true, dataContext); + var mkdir = storage.mkdir("testMkdir_unicode/你好/你好", true, dataContext); waitFor(TEST_TIMEOUT, mkdir); Assertions.assertDoesNotThrow(() -> getResultOf(mkdir)); - var dirExists = storage.exists("你好", dataContext); - var childExists = storage.exists("你好/你好", dataContext); + var dirExists = storage.exists("testMkdir_unicode/你好", dataContext); + var childExists = storage.exists("testMkdir_unicode/你好/你好", dataContext); waitFor(TEST_TIMEOUT, dirExists, childExists); Assertions.assertTrue(getResultOf(dirExists)); @@ -921,21 +937,21 @@ void testRm_ok() throws Exception { // Simplest case - create one file and delete it - var prepare = makeSmallFile("test_file.txt", storage, dataContext); + var prepare = makeSmallFile("testRm_ok.txt", storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var exists1 = storage.exists("test_file.txt", dataContext); + var exists1 = storage.exists("testRm_ok.txt", dataContext); waitFor(TEST_TIMEOUT, exists1); Assertions.assertTrue(getResultOf(exists1)); - var rm = storage.rm("test_file.txt", dataContext); + var rm = storage.rm("testRm_ok.txt", dataContext); waitFor(TEST_TIMEOUT, rm); Assertions.assertDoesNotThrow(() -> getResultOf(rm)); // File should be gone - var exists2 = storage.exists("test_file.txt", dataContext); + var exists2 = storage.exists("testRm_ok.txt", dataContext); waitFor(TEST_TIMEOUT, exists2); Assertions.assertFalse(getResultOf(exists2)); } @@ -943,23 +959,25 @@ void testRm_ok() throws Exception { @Test void testRm_longPath() throws Exception { + var longPath = makeLongPath("testRm_longPath", ".txt"); + // Simplest case - create one file and delete it - var prepare = makeSmallFile(LONG_PATH_TXT_FILE, storage, dataContext); + var prepare = makeSmallFile(longPath, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var exists1 = storage.exists(LONG_PATH_TXT_FILE, dataContext); + var exists1 = storage.exists(longPath, dataContext); waitFor(TEST_TIMEOUT, exists1); Assertions.assertTrue(getResultOf(exists1)); - var rm = storage.rm(LONG_PATH_TXT_FILE, dataContext); + var rm = storage.rm(longPath, dataContext); waitFor(TEST_TIMEOUT, rm); Assertions.assertDoesNotThrow(() -> getResultOf(rm)); // File should be gone - var exists2 = storage.exists(LONG_PATH_TXT_FILE, dataContext); + var exists2 = storage.exists(longPath, dataContext); waitFor(TEST_TIMEOUT, exists2); Assertions.assertFalse(getResultOf(exists2)); } @@ -969,17 +987,17 @@ void testRm_inSubdirOk() throws Exception { // Simplest case - create one file and delete it - var prepare = makeSmallFile("sub_dir/test_file.txt", storage, dataContext); + var prepare = makeSmallFile("testRm_inSubdirOk/test_file.txt", storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var rm = storage.rm("sub_dir/test_file.txt", dataContext); + var rm = storage.rm("testRm_inSubdirOk/test_file.txt", dataContext); waitFor(TEST_TIMEOUT, rm); Assertions.assertDoesNotThrow(() -> getResultOf(rm)); // File should be gone - var exists = storage.exists("sub_dir/test_file.txt", dataContext); + var exists = storage.exists("testRm_inSubdirOk/test_file.txt", dataContext); waitFor(TEST_TIMEOUT, exists); Assertions.assertFalse(getResultOf(exists)); } @@ -989,17 +1007,17 @@ void testRm_onDir() throws Exception { // Calling rm on a directory is a bad request, even if the dir is empty - var prepare = storage.mkdir("test_dir", false, dataContext); + var prepare = storage.mkdir("testRm_onDir", false, dataContext); waitFor(TEST_TIMEOUT, prepare); - var rm = storage.rm("test_dir", dataContext); + var rm = storage.rm("testRm_onDir", dataContext); waitFor(TEST_TIMEOUT, rm); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(rm)); // Dir should still exist because rm has failed - var exists = storage.exists("test_dir", dataContext); + var exists = storage.exists("testRm_onDir", dataContext); waitFor(TEST_TIMEOUT, exists); Assertions.assertTrue(getResultOf(exists)); } @@ -1009,7 +1027,7 @@ void testRm_missing() { // Try to delete a path that does not exist - var rm = storage.rm("missing_path.dat", dataContext); + var rm = storage.rm("testRm_missing.dat", dataContext); waitFor(TEST_TIMEOUT, rm); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(rm)); @@ -1035,18 +1053,18 @@ void testRm_storageRoot() { @Test void testRmdir_ok() throws Exception { - var prepare = storage.mkdir("test_dir", false, dataContext); + var prepare = storage.mkdir("testRmdir_ok", false, dataContext); waitFor(TEST_TIMEOUT, prepare); - var exists1 = storage.exists("test_dir", dataContext); + var exists1 = storage.exists("testRmdir_ok", dataContext); waitFor(TEST_TIMEOUT, exists1); Assertions.assertTrue(getResultOf(exists1)); - var rmdir = storage.rmdir("test_dir", dataContext); + var rmdir = storage.rmdir("testRmdir_ok", dataContext); waitFor(TEST_TIMEOUT, rmdir); Assertions.assertDoesNotThrow(() -> getResultOf(rmdir)); - var exists2 = storage.exists("test_dir", dataContext); + var exists2 = storage.exists("testRmdir_ok", dataContext); waitFor(TEST_TIMEOUT, exists2); Assertions.assertFalse(getResultOf(exists2)); } @@ -1054,18 +1072,20 @@ void testRmdir_ok() throws Exception { @Test void testRmdir_longPath() throws Exception { - var prepare = storage.mkdir(LONG_PATH_DIR, false, dataContext); + var longPath = makeLongDirPath("testRmdir_longPath"); + + var prepare = storage.mkdir(longPath, false, dataContext); waitFor(TEST_TIMEOUT, prepare); - var exists1 = storage.exists(LONG_PATH_DIR, dataContext); + var exists1 = storage.exists(longPath, dataContext); waitFor(TEST_TIMEOUT, exists1); Assertions.assertTrue(getResultOf(exists1)); - var rmdir = storage.rmdir(LONG_PATH_DIR, dataContext); + var rmdir = storage.rmdir(longPath, dataContext); waitFor(TEST_TIMEOUT, rmdir); Assertions.assertDoesNotThrow(() -> getResultOf(rmdir)); - var exists2 = storage.exists(LONG_PATH_DIR, dataContext); + var exists2 = storage.exists(longPath, dataContext); waitFor(TEST_TIMEOUT, exists2); Assertions.assertFalse(getResultOf(exists2)); } @@ -1073,18 +1093,18 @@ void testRmdir_longPath() throws Exception { @Test void testRmdir_byPrefix() throws Exception { - var prepare = storage.mkdir("test_dir/sub_dir", true, dataContext); + var prepare = storage.mkdir("testRmdir_byPrefix/sub_dir", true, dataContext); waitFor(TEST_TIMEOUT, prepare); - var exists1 = storage.exists("test_dir", dataContext); + var exists1 = storage.exists("testRmdir_byPrefix", dataContext); waitFor(TEST_TIMEOUT, exists1); Assertions.assertTrue(getResultOf(exists1)); - var rmdir = storage.rmdir("test_dir", dataContext); + var rmdir = storage.rmdir("testRmdir_byPrefix", dataContext); waitFor(TEST_TIMEOUT, rmdir); Assertions.assertDoesNotThrow(() -> getResultOf(rmdir)); - var exists2 = storage.exists("test_dir", dataContext); + var exists2 = storage.exists("testRmdir_byPrefix", dataContext); waitFor(TEST_TIMEOUT, exists2); Assertions.assertFalse(getResultOf(exists2)); } @@ -1096,22 +1116,22 @@ void testRmdir_withContent() throws Exception { // Sibling dir tree should be unaffected var prepare = storage - .mkdir("test_dir/child_1", true, dataContext) - .thenCompose(x -> storage.mkdir("test_dir/child_1/sub", false, dataContext)) - .thenCompose(x -> makeSmallFile("test_dir/child_1/file_a.txt", storage, dataContext)) - .thenCompose(x -> makeSmallFile("test_dir/child_1/file_b.txt", storage, dataContext)) - .thenCompose(x -> storage.mkdir("test_dir/child_2", true, dataContext)) - .thenCompose(x -> makeSmallFile("test_dir/child_2/file_a.txt", storage, dataContext)); + .mkdir("testRmdir_withContent/child_1", true, dataContext) + .thenCompose(x -> storage.mkdir("testRmdir_withContent/child_1/sub", false, dataContext)) + .thenCompose(x -> makeSmallFile("testRmdir_withContent/child_1/file_a.txt", storage, dataContext)) + .thenCompose(x -> makeSmallFile("testRmdir_withContent/child_1/file_b.txt", storage, dataContext)) + .thenCompose(x -> storage.mkdir("testRmdir_withContent/child_2", true, dataContext)) + .thenCompose(x -> makeSmallFile("testRmdir_withContent/child_2/file_a.txt", storage, dataContext)); waitFor(TEST_TIMEOUT.multipliedBy(2), prepare); // Allow extra time for multiple operations - var rmdir = storage.rmdir("test_dir/child_1", dataContext); + var rmdir = storage.rmdir("testRmdir_withContent/child_1", dataContext); waitFor(TEST_TIMEOUT, rmdir); Assertions.assertDoesNotThrow(() -> getResultOf(rmdir)); - var exists1 = storage.exists("test_dir/child_1", dataContext); - var exists2 = storage.exists("test_dir/child_2", dataContext); - var size2a = storage.size("test_dir/child_2/file_a.txt", dataContext); + var exists1 = storage.exists("testRmdir_withContent/child_1", dataContext); + var exists2 = storage.exists("testRmdir_withContent/child_2", dataContext); + var size2a = storage.size("testRmdir_withContent/child_2/file_a.txt", dataContext); waitFor(TEST_TIMEOUT, exists1, exists2, size2a); Assertions.assertFalse(getResultOf(exists1)); @@ -1124,16 +1144,16 @@ void testRmdir_onFile() throws Exception { // Calling rmdir on a file is a bad request - var prepare = makeSmallFile("test_file.txt", storage, dataContext); + var prepare = makeSmallFile("testRmdir_onFile.txt", storage, dataContext); waitFor(TEST_TIMEOUT, prepare); - var rmdir = storage.rmdir("test_file.txt", dataContext); + var rmdir = storage.rmdir("testRmdir_onFile.txt", dataContext); waitFor(TEST_TIMEOUT, rmdir); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(rmdir)); // File should still exist because rm has failed - var exists = storage.exists("test_file.txt", dataContext); + var exists = storage.exists("testRmdir_onFile.txt", dataContext); waitFor(TEST_TIMEOUT, exists); Assertions.assertTrue(getResultOf(exists)); } @@ -1143,7 +1163,7 @@ void testRmdir_Missing() { // Try to delete a path that does not exist - var rmdir = storage.rmdir("missing_path", dataContext); + var rmdir = storage.rmdir("testRmdir_Missing", dataContext); waitFor(TEST_TIMEOUT, rmdir); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(rmdir)); } @@ -1174,12 +1194,12 @@ void testReadChunk_ok() throws Exception { random.nextBytes(bytes); var content = Bytes.copyToBuffer(bytes, dataContext.arrowAllocator()); - var prepare = makeFile("test_file.dat", content, storage, dataContext); + var prepare = makeFile("testReadChunk_ok.dat", content, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); getResultOf(prepare); - var readChunk = storage.readChunk("test_file.dat", 4096, 4096, dataContext); + var readChunk = storage.readChunk("testReadChunk_ok.dat", 4096, 4096, dataContext); waitFor(TEST_TIMEOUT, readChunk); try (var chunk = getResultOf(readChunk)) { @@ -1201,12 +1221,12 @@ void testReadChunk_first() throws Exception { random.nextBytes(bytes); var content = Bytes.copyToBuffer(bytes, dataContext.arrowAllocator()); - var prepare = makeFile("test_file.dat", content, storage, dataContext); + var prepare = makeFile("testReadChunk_first.dat", content, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); getResultOf(prepare); - var readChunk = storage.readChunk("test_file.dat", 0, 4096, dataContext); + var readChunk = storage.readChunk("testReadChunk_first.dat", 0, 4096, dataContext); waitFor(TEST_TIMEOUT, readChunk); try (var chunk = getResultOf(readChunk)) { @@ -1228,12 +1248,12 @@ void testReadChunk_last() throws Exception { random.nextBytes(bytes); var content = Bytes.copyToBuffer(bytes, dataContext.arrowAllocator()); - var prepare = makeFile("test_file.dat", content, storage, dataContext); + var prepare = makeFile("testReadChunk_last.dat", content, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); getResultOf(prepare); - var readChunk = storage.readChunk("test_file.dat", 8192, 2048, dataContext); + var readChunk = storage.readChunk("testReadChunk_last.dat", 8192, 2048, dataContext); waitFor(TEST_TIMEOUT, readChunk); try (var chunk = getResultOf(readChunk)) { @@ -1255,12 +1275,12 @@ void testReadChunk_all() throws Exception { random.nextBytes(bytes); var content = Bytes.copyToBuffer(bytes, dataContext.arrowAllocator()); - var prepare = makeFile("test_file.dat", content, storage, dataContext); + var prepare = makeFile("testReadChunk_all.dat", content, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); getResultOf(prepare); - var readChunk = storage.readChunk("test_file.dat", 0, 10240, dataContext); + var readChunk = storage.readChunk("testReadChunk_all.dat", 0, 10240, dataContext); waitFor(TEST_TIMEOUT, readChunk); try (var chunk = getResultOf(readChunk)) { @@ -1282,17 +1302,17 @@ void testReadChunk_badSize() throws Exception { random.nextBytes(bytes); var content = Bytes.copyToBuffer(bytes, dataContext.arrowAllocator()); - var prepare = makeFile("test_file.dat", content, storage, dataContext); + var prepare = makeFile("testReadChunk_badSize.dat", content, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); getResultOf(prepare); - var readChunk = storage.readChunk("test_file.dat", 1024, 10240, dataContext); + var readChunk = storage.readChunk("testReadChunk_badSize.dat", 1024, 10240, dataContext); waitFor(TEST_TIMEOUT, readChunk); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(readChunk)); - var readChunk2 = storage.readChunk("test_file.dat", 10241, 1024, dataContext); + var readChunk2 = storage.readChunk("testReadChunk_badSize.dat", 10241, 1024, dataContext); waitFor(TEST_TIMEOUT, readChunk2); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(readChunk2)); @@ -1303,11 +1323,11 @@ void testReadChunk_onDir() throws Exception { // Calling readChunk on a directory is a storage error - var prepare = storage.mkdir("test_file.dat", false, dataContext); + var prepare = storage.mkdir("testReadChunk_onDir.dat", false, dataContext); waitFor(TEST_TIMEOUT, prepare); getResultOf(prepare); - var readChunk = storage.readChunk("test_file.dat", 0, 1024, dataContext); + var readChunk = storage.readChunk("testReadChunk_onDir.dat", 0, 1024, dataContext); waitFor(TEST_TIMEOUT, readChunk); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(readChunk)); @@ -1318,7 +1338,7 @@ void testReadChunk_missing() { // Calling readChunk on a missing file is a storage error - var readChunk = storage.readChunk("missing_file.dat", 0, 1024, dataContext); + var readChunk = storage.readChunk("testReadChunk_missing.dat", 0, 1024, dataContext); waitFor(TEST_TIMEOUT, readChunk); Assertions.assertThrows(EStorageRequest.class, () -> getResultOf(readChunk)); } @@ -1332,26 +1352,26 @@ void testReadChunk_invalidParams() throws Exception { random.nextBytes(bytes); var content = Bytes.copyToBuffer(bytes, dataContext.arrowAllocator()); - var prepare = makeFile("test_file.dat", content, storage, dataContext); + var prepare = makeFile("testReadChunk_invalidParams.dat", content, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); getResultOf(prepare); // Zero size (empty chunk) - var readChunk = storage.readChunk("test_file.dat", 1024, 0, dataContext); + var readChunk = storage.readChunk("testReadChunk_invalidParams.dat", 1024, 0, dataContext); waitFor(TEST_TIMEOUT, readChunk); Assertions.assertThrows(EValidationGap.class, () -> getResultOf(readChunk)); // Negative size - var readChunk3 = storage.readChunk("test_file.dat", 0, -1024, dataContext); + var readChunk3 = storage.readChunk("testReadChunk_invalidParams.dat", 0, -1024, dataContext); waitFor(TEST_TIMEOUT, readChunk3); Assertions.assertThrows(EValidationGap.class, () -> getResultOf(readChunk3)); // Negative offset - var readChunk2 = storage.readChunk("test_file.dat", -1, 1024, dataContext); + var readChunk2 = storage.readChunk("testReadChunk_invalidParams.dat", -1, 1024, dataContext); waitFor(TEST_TIMEOUT, readChunk2); Assertions.assertThrows(EValidationGap.class, () -> getResultOf(readChunk2)); } diff --git a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageReadOnlyTestSuite.java b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageReadOnlyTestSuite.java index f69f94ed6..cb62c9517 100644 --- a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageReadOnlyTestSuite.java +++ b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageReadOnlyTestSuite.java @@ -47,11 +47,11 @@ public abstract class StorageReadOnlyTestSuite { @Test void testMkdir_readOnly() throws Exception { - var dir = roStorage.mkdir("test_dir", false, dataContext); + var dir = roStorage.mkdir("testMkdir_readOnly", false, dataContext); waitFor(TEST_TIMEOUT, dir); Assertions.assertThrows(EStorageAccess.class, () -> getResultOf(dir)); - var dirPresent = roStorage.exists("test_dir", dataContext); + var dirPresent = roStorage.exists("testMkdir_readOnly", dataContext); waitFor(TEST_TIMEOUT, dirPresent); Assertions.assertFalse(getResultOf(dirPresent)); } @@ -63,11 +63,11 @@ void testRm_readOnly() throws Exception { // The file is created independently of the storage, because the storage is assumed to be read only // and the file cannot be created in it. - var prepare = makeSmallFile("test_file.txt", rwStorage, dataContext); + var prepare = makeSmallFile("testRm_readOnly.txt", rwStorage, dataContext); waitFor(TEST_TIMEOUT, prepare); getResultOf(prepare); - var created = roStorage.exists("test_file.txt", dataContext); + var created = roStorage.exists("testRm_readOnly.txt", dataContext); waitFor(TEST_TIMEOUT, created); Assertions.assertTrue(getResultOf(created)); @@ -77,7 +77,7 @@ void testRm_readOnly() throws Exception { // File should not be gone - var exists = roStorage.exists("test_file.txt", dataContext); + var exists = roStorage.exists("testRm_readOnly.txt", dataContext); waitFor(TEST_TIMEOUT, exists); Assertions.assertTrue(getResultOf(exists)); } @@ -89,21 +89,21 @@ void testRmdir_readOnly() throws Exception { // The file is created independently of the storage, because the storage is assumed to be read only // and the file cannot be created in it. - var prepare = rwStorage.mkdir("test_dir", false, dataContext); + var prepare = rwStorage.mkdir("testRmdir_readOnly", false, dataContext); waitFor(TEST_TIMEOUT, prepare); getResultOf(prepare); - var created = roStorage.exists("test_dir", dataContext); + var created = roStorage.exists("testRmdir_readOnly", dataContext); waitFor(TEST_TIMEOUT, created); Assertions.assertTrue(getResultOf(created)); - var rmdir = roStorage.rmdir("test_dir", dataContext); + var rmdir = roStorage.rmdir("testRmdir_readOnly", dataContext); waitFor(TEST_TIMEOUT, rmdir); Assertions.assertThrows(EStorageAccess.class, () -> getResultOf(rmdir)); // Dir should not be gone - var exists = roStorage.exists("test_dir", dataContext); + var exists = roStorage.exists("testRmdir_readOnly", dataContext); waitFor(TEST_TIMEOUT, exists); Assertions.assertTrue(getResultOf(exists)); } @@ -111,7 +111,7 @@ void testRmdir_readOnly() throws Exception { @Test void testWriter_readOnly() { - var storagePath = "any_file.txt"; + var storagePath = "testWriter_readOnly.txt"; var writeSignal = new CompletableFuture(); var writer = roStorage.writer(storagePath, writeSignal, dataContext); diff --git a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageReadWriteTestSuite.java b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageReadWriteTestSuite.java index 517621537..e95f4b5aa 100644 --- a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageReadWriteTestSuite.java +++ b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/StorageReadWriteTestSuite.java @@ -81,7 +81,7 @@ public abstract class StorageReadWriteTestSuite { @Test void roundTrip_basic() throws Exception { - var storagePath = "haiku.txt"; + var storagePath = "roundTrip_basic.txt"; var haiku = "The data goes in;\n" + @@ -109,7 +109,7 @@ void roundTrip_long_path() throws Exception { @Test void roundTrip_large() throws Exception { - var storagePath = "test_file.dat"; + var storagePath = "roundTrip_large.dat"; var bytes = new byte[10 * 1024 * 1024]; // One 10 M chunk @@ -124,7 +124,7 @@ void roundTrip_large() throws Exception { @Test void roundTrip_heterogeneous() throws Exception { - var storagePath = "test_file.dat"; + var storagePath = "roundTrip_heterogeneous.dat"; var bytes = List.of( // Selection of different size chunks new byte[3], @@ -145,7 +145,7 @@ void roundTrip_heterogeneous() throws Exception { @Test void roundTrip_empty() throws Exception { - var storagePath = "test_file.dat"; + var storagePath = "roundTrip_empty.dat"; var emptyBytes = new byte[0]; StorageReadWriteTestSuite.roundTripTest( @@ -230,7 +230,7 @@ void testWrite_missingDir() throws Exception { // Basic test without creating the parent dir first // Should be automatically created by the writer - var storagePath = "parent_dir/haiku.txt"; + var storagePath = "testWrite_missingDir/haiku.txt"; var haiku = "The data goes in;\n" + @@ -248,7 +248,7 @@ void testWrite_fileAlreadyExists() throws Exception { // Writing a file always overwrites any existing content // This is in line with cloud bucket semantics - var storagePath = "some_file.txt"; + var storagePath = "testWrite_fileAlreadyExists.txt"; var prepare = makeSmallFile(storagePath, storage, dataContext); waitFor(TEST_TIMEOUT, prepare); @@ -270,7 +270,7 @@ void testWrite_dirAlreadyExists() throws Exception { // File storage should not allow a file to be written if a dir exists with the same name // TRAC prohibits this even though it is allowed in pure bucket semantics - var storagePath = "some_file.txt"; + var storagePath = "testWrite_dirAlreadyExists.txt"; var prepare = storage.mkdir(storagePath, false, dataContext); waitFor(TEST_TIMEOUT, prepare); @@ -280,32 +280,27 @@ void testWrite_dirAlreadyExists() throws Exception { var exists1Result = getResultOf(exists1); Assertions.assertTrue(exists1Result); - var content = Bytes.copyToBuffer( - "Some content".getBytes(StandardCharsets.UTF_8), - dataContext.arrowAllocator()); + var content = "Some content".getBytes(StandardCharsets.UTF_8); + var contentStream = Stream + .of(content) + .map(chunk -> Bytes.copyToBuffer(chunk, dataContext.arrowAllocator())); - var contentStream = Flows.publish(Stream.of(content)); var writeSignal = new CompletableFuture(); var writer = storage.writer(storagePath, writeSignal, dataContext); - contentStream.subscribe(writer); - waitFor(TEST_TIMEOUT, writeSignal); - - var exists = storage.exists(storagePath, dataContext); - waitFor(TEST_TIMEOUT, exists); + Assertions.assertThrows(EStorageRequest.class, () -> { - Assertions.assertTrue(getResultOf(exists)); + Flows.publish(contentStream).subscribe(writer); + waitFor(TEST_TIMEOUT, writeSignal); + getResultOf(writeSignal); + }); - var contentStream2 = Flows.publish(Stream.of(content)); - var writeSignal2 = new CompletableFuture(); - var writer2 = storage.writer(storagePath, writeSignal2, dataContext); + // Dir should still exist after write failure - Assertions.assertThrows(EStorageRequest.class, () -> { + var stat = storage.stat(storagePath, dataContext); + waitFor(TEST_TIMEOUT, stat); - contentStream2.subscribe(writer2); - waitFor(TEST_TIMEOUT, writeSignal2); - getResultOf(writeSignal2); - }); + Assertions.assertEquals(FileType.DIRECTORY, getResultOf(stat).fileType); } @Test @@ -346,7 +341,7 @@ void testWrite_storageRoot() { @Test void testWrite_outsideRoot() { - var storagePath = "../any_file.txt"; + var storagePath = "../testWrite_outsideRoot.txt"; var writeSignal = new CompletableFuture(); @@ -357,7 +352,7 @@ void testWrite_outsideRoot() { @Test void testRead_missing() { - var storagePath = "missing_file.txt"; + var storagePath = "testRead_missing.txt"; var reader = storage.reader(storagePath, dataContext); var result = new ArrayList(); @@ -409,7 +404,7 @@ void testRead_storageRoot() { @Test void testRead_outsideRoot() { - var storagePath = "../some_file.txt"; + var storagePath = "../testRead_outsideRoot.txt"; Assertions.assertThrows(EValidationGap.class, () -> storage.reader(storagePath, dataContext)); @@ -445,7 +440,7 @@ void testWrite_chunksReleased() throws Exception { // Writer takes ownership of chunks when it receives them // This test makes sure they are being released after they have been written - var storagePath = "test_file.dat"; + var storagePath = "testWrite_chunksReleased.dat"; var bytes = List.of( new byte[3], @@ -482,16 +477,16 @@ void testWrite_subscribeNever() throws Exception { // Set up a dir in storage - var mkdir = storage.mkdir("some_dir", false, dataContext); + var mkdir = storage.mkdir("testWrite_subscribeNever", false, dataContext); waitFor(TEST_TIMEOUT, mkdir); - var dirExists = storage.exists("some_dir", dataContext); + var dirExists = storage.exists("testWrite_subscribeNever", dataContext); waitFor(TEST_TIMEOUT, dirExists); Assertions.assertTrue(getResultOf(dirExists)); // Prepare a writer to write a file inside the new dir - var storagePath = "some_dir/some_file.txt"; + var storagePath = "testWrite_subscribeNever/some_file.txt"; var writeSignal = new CompletableFuture(); storage.writer(storagePath, writeSignal, dataContext); @@ -507,7 +502,7 @@ void testWrite_subscribeNever() throws Exception { @Test void testWrite_subscribeTwice() throws Exception { - var storagePath = "test_file.dat"; + var storagePath = "testWrite_subscribeTwice.dat"; var bytes = new byte[10000]; new Random().nextBytes(bytes); @@ -543,7 +538,7 @@ void testWrite_subscribeTwice() throws Exception { @Test void testWrite_errorImmediately() throws Exception { - var storagePath = "test_file.dat"; + var storagePath = "testWrite_errorImmediately.dat"; var writerSignal = new CompletableFuture(); var writer = storage.writer(storagePath, writerSignal, dataContext); @@ -590,7 +585,7 @@ void testWrite_errorImmediately() throws Exception { @Test void testWrite_errorAfterChunk() throws Exception { - var storagePath = "test_file.dat"; + var storagePath = "testWrite_errorAfterChunk.dat"; var bytes0 = new byte[10000]; new Random().nextBytes(bytes0); @@ -646,7 +641,7 @@ void testWrite_errorAfterChunk() throws Exception { @Test void testWrite_errorThenRetry() throws Exception { - var storagePath = "test_file.dat"; + var storagePath = "testWrite_errorThenRetry.dat"; var dataSize = 10000; var bytes = new byte[dataSize]; @@ -704,7 +699,7 @@ void testRead_requestUpfront() { try { - var storagePath = "some_file.dat"; + var storagePath = "testRead_requestUpfront.dat"; // Create a file big enough that it needs many chunks to read @@ -770,7 +765,7 @@ void testRead_requestUpfront() { @Test void testRead_subscribeLate() throws Exception { - var storagePath = "some_file.txt"; + var storagePath = "testRead_subscribeLate.txt"; var writeSignal = makeSmallFile(storagePath, storage, dataContext); waitFor(TEST_TIMEOUT, writeSignal); @@ -800,7 +795,7 @@ void testRead_subscribeLate() throws Exception { @Test void testRead_subscribeTwice() throws Exception { - var storagePath = "some_file.txt"; + var storagePath = "testRead_subscribeTwice.txt"; var writeSignal = makeSmallFile(storagePath, storage, dataContext); waitFor(TEST_TIMEOUT, writeSignal); @@ -847,7 +842,7 @@ void testRead_subscribeTwice() throws Exception { @Test void testRead_cancelImmediately() { - var storagePath = "some_file.txt"; + var storagePath = "testRead_cancelImmediately.txt"; var writeSignal = makeSmallFile(storagePath, storage, dataContext); waitFor(TEST_TIMEOUT, writeSignal); @@ -882,7 +877,7 @@ void testRead_cancelImmediately() { @Test void testRead_cancelAndRetry() throws Exception { - var storagePath = "some_file.dat"; + var storagePath = "testRead_cancelAndRetry.dat"; // Create a file big enough that it can't be read in a single chunk @@ -949,7 +944,7 @@ void testRead_cancelAndRetry() throws Exception { @Test void testRead_cancelAndDelete() throws Exception { - var storagePath = "some_file.dat"; + var storagePath = "testRead_cancelAndDelete.dat"; // Create a file big enough that it can't be read in a single chunk diff --git a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageOperationsTest.java b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageOperationsTest.java index ae33147d8..2080c91e5 100644 --- a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageOperationsTest.java +++ b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageOperationsTest.java @@ -16,6 +16,7 @@ package org.finos.tracdap.common.storage.local; +import org.apache.arrow.memory.BufferAllocator; import org.finos.tracdap.common.data.DataContext; import org.finos.tracdap.common.storage.IStorageManager; import org.finos.tracdap.common.storage.StorageOperationsTestSuite; @@ -24,6 +25,8 @@ import io.netty.util.concurrent.DefaultThreadFactory; import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; @@ -34,17 +37,39 @@ public class LocalStorageOperationsTest extends StorageOperationsTestSuite { @TempDir - Path storageDir; + static Path storageDir; + + static BufferAllocator allocator; + + static LocalFileStorage storageInstance; + static DataContext contextInstance; + + @BeforeAll + static void setupStorage() { - @BeforeEach - void setupStorage() { var storageProps = new Properties(); storageProps.put(IStorageManager.PROP_STORAGE_KEY, "TEST_STORAGE"); storageProps.put(LocalFileStorage.CONFIG_ROOT_PATH, storageDir.toString()); - storage = new LocalFileStorage("TEST_STORAGE", storageProps); + storageInstance = new LocalFileStorage("TEST_STORAGE", storageProps); + + allocator = new RootAllocator(); var elExecutor = new DefaultEventExecutor(new DefaultThreadFactory("t-events")); - dataContext = new DataContext(elExecutor, new RootAllocator()); + contextInstance = new DataContext(elExecutor, allocator); + } + + @BeforeEach + void useStorageInstance() { + + storage = storageInstance; + dataContext = contextInstance; + } + + @AfterAll + static void tearDownStorage() { + + storageInstance.close(); + allocator.close(); } } diff --git a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageReadOnlyTest.java b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageReadOnlyTest.java index f1d47c179..d2afb9365 100644 --- a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageReadOnlyTest.java +++ b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageReadOnlyTest.java @@ -18,12 +18,15 @@ import io.netty.util.concurrent.DefaultEventExecutor; import io.netty.util.concurrent.DefaultThreadFactory; +import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.finos.tracdap.common.data.DataContext; import org.finos.tracdap.common.storage.CommonFileStorage; import org.finos.tracdap.common.storage.IStorageManager; import org.finos.tracdap.common.storage.StorageReadOnlyTestSuite; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; @@ -33,20 +36,47 @@ public class LocalStorageReadOnlyTest extends StorageReadOnlyTestSuite { - @BeforeEach - void setupStorage(@TempDir Path storageDir) { + @TempDir + static Path storageDir; + + static BufferAllocator allocator; + + static LocalFileStorage rwStorageInstance; + static LocalFileStorage roStorageInstance; + static DataContext contextInstance; + + @BeforeAll + static void setupStorage() { var rwProps = new Properties(); rwProps.put(IStorageManager.PROP_STORAGE_KEY, "TEST_STORAGE_NOT_WRITABLE"); rwProps.put(LocalFileStorage.CONFIG_ROOT_PATH, storageDir.toString()); - rwStorage = new LocalFileStorage("TEST_LOCAL_RW_STORAGE", rwProps); + rwStorageInstance = new LocalFileStorage("TEST_LOCAL_RW_STORAGE", rwProps); var roProps = new Properties(); roProps.putAll(rwProps); roProps.put(CommonFileStorage.READ_ONLY_CONFIG_KEY, "true"); - roStorage = new LocalFileStorage("TEST_LOCAL_RO_STORAGE", roProps); + roStorageInstance = new LocalFileStorage("TEST_LOCAL_RO_STORAGE", roProps); + + allocator = new RootAllocator(); + + var elExecutor = new DefaultEventExecutor(new DefaultThreadFactory("t-events")); + contextInstance = new DataContext(elExecutor, allocator); + } + + @BeforeEach + void useStorageInstance() { + + rwStorage = rwStorageInstance; + roStorage = roStorageInstance; + dataContext = contextInstance; + } + + @AfterAll + static void tearDownStorage() { - var executor = new DefaultEventExecutor(new DefaultThreadFactory("t-events")); - dataContext = new DataContext(executor, new RootAllocator()); + roStorageInstance.close(); + rwStorageInstance.close(); + allocator.close(); } } diff --git a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageReadWriteTest.java b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageReadWriteTest.java index 82999ca61..a08d363e9 100644 --- a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageReadWriteTest.java +++ b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageReadWriteTest.java @@ -18,10 +18,13 @@ import io.netty.util.concurrent.DefaultEventExecutor; import io.netty.util.concurrent.DefaultThreadFactory; +import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.finos.tracdap.common.data.DataContext; import org.finos.tracdap.common.storage.StorageReadWriteTestSuite; import org.finos.tracdap.common.storage.IStorageManager; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; @@ -32,17 +35,39 @@ public class LocalStorageReadWriteTest extends StorageReadWriteTestSuite { @TempDir - Path storageDir; + static Path storageDir; + + static BufferAllocator allocator; + + static LocalFileStorage storageInstance; + static DataContext contextInstance; + + @BeforeAll + static void setupStorage() { - @BeforeEach - void setupStorage() { var storageProps = new Properties(); storageProps.put(IStorageManager.PROP_STORAGE_KEY, "TEST_STORAGE"); storageProps.put(LocalFileStorage.CONFIG_ROOT_PATH, storageDir.toString()); - storage = new LocalFileStorage("TEST_STORAGE", storageProps); + storageInstance = new LocalFileStorage("TEST_STORAGE", storageProps); + + allocator = new RootAllocator(); + + var elExecutor = new DefaultEventExecutor(new DefaultThreadFactory("t-events")); + contextInstance = new DataContext(elExecutor, allocator); + } + + @BeforeEach + void useStorageInstance() { + + storage = storageInstance; + dataContext = contextInstance; + } + + @AfterAll + static void tearDownStorage() { - var executor = new DefaultEventExecutor(new DefaultThreadFactory("t-events")); - dataContext = new DataContext(executor, new RootAllocator()); + storageInstance.close(); + allocator.close(); } } diff --git a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageStabilityTest.java b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageStabilityTest.java index 17af7ac84..9b384b39c 100644 --- a/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageStabilityTest.java +++ b/tracdap-libs/tracdap-lib-data/src/test/java/org/finos/tracdap/common/storage/local/LocalStorageStabilityTest.java @@ -16,6 +16,7 @@ package org.finos.tracdap.common.storage.local; +import org.apache.arrow.memory.BufferAllocator; import org.finos.tracdap.common.data.DataContext; import org.finos.tracdap.common.storage.StorageStabilityTestSuite; import org.finos.tracdap.common.storage.IStorageManager; @@ -24,7 +25,9 @@ import io.netty.util.concurrent.DefaultThreadFactory; import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.io.TempDir; import java.nio.file.Path; @@ -36,15 +39,37 @@ public class LocalStorageStabilityTest extends StorageStabilityTestSuite { @TempDir static Path storageDir; + static BufferAllocator allocator; + + static LocalFileStorage storageInstance; + static DataContext contextInstance; + @BeforeAll static void setupStorage() { + var storageProps = new Properties(); storageProps.put(IStorageManager.PROP_STORAGE_KEY, "TEST_STORAGE"); storageProps.put(LocalFileStorage.CONFIG_ROOT_PATH, storageDir.toString()); - storage = new LocalFileStorage("TEST_STORAGE", storageProps); + storageInstance = new LocalFileStorage("TEST_STORAGE", storageProps); + + allocator = new RootAllocator(); + + var elExecutor = new DefaultEventExecutor(new DefaultThreadFactory("t-events")); + contextInstance = new DataContext(elExecutor, allocator); + } + + @BeforeEach + void useStorageInstance() { + + storage = storageInstance; + dataContext = contextInstance; + } + + @AfterAll + static void tearDownStorage() { - var executor = new DefaultEventExecutor(new DefaultThreadFactory("t-events")); - dataContext = new DataContext(executor, new RootAllocator()); + storageInstance.close(); + allocator.close(); } } diff --git a/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageOperationsTest.java b/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageOperationsTest.java index 687ade4f2..6a32d2f4b 100644 --- a/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageOperationsTest.java +++ b/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageOperationsTest.java @@ -42,7 +42,7 @@ @Tag("aws-platform") public class S3StorageOperationsTest extends StorageOperationsTestSuite { - static Duration SETUP_TIMEOUT = Duration.of(5, ChronoUnit.SECONDS); + static Duration SETUP_TIMEOUT = Duration.of(10, ChronoUnit.SECONDS); static Properties storageProps; static String testSuiteDir; @@ -50,8 +50,8 @@ public class S3StorageOperationsTest extends StorageOperationsTestSuite { static EventLoopGroup elg; static BufferAllocator allocator; - static DataContext setupCtx; - static S3ObjectStorage setupStorage; + static DataContext setupCtx, testCtx; + static S3ObjectStorage setupStorage, testStorage; static int testNumber; @@ -74,44 +74,35 @@ static void setupStorage() throws Exception { setupStorage.start(elg); var mkdir = setupStorage.mkdir(testSuiteDir, true, setupCtx); - waitFor(Duration.ofSeconds(10), mkdir); - resultOf(mkdir); - } - - @BeforeEach - void setup() throws Exception { - - var testDir = String.format("%stest_%d", testSuiteDir, ++testNumber); - - var mkdir = setupStorage.mkdir(testDir, false, setupCtx); waitFor(SETUP_TIMEOUT, mkdir); resultOf(mkdir); - storageProps.put(S3ObjectStorage.PREFIX_PROPERTY, testDir); - storage = new S3ObjectStorage("TEST_" + testNumber, storageProps); - storage.start(elg); + storageProps.put(S3ObjectStorage.PREFIX_PROPERTY, testSuiteDir); + testStorage = new S3ObjectStorage("TEST_" + testNumber, storageProps); + testStorage.start(elg); - dataContext = new DataContext(elg.next(), allocator); + testCtx = new DataContext(elg.next(), allocator); } - @AfterEach - void tearDown() { + @BeforeEach + void setup() { - storage.stop(); - storage = null; + storage = testStorage; + dataContext = testCtx; } @AfterAll static void tearDownStorage() throws Exception { + testStorage.stop(); + var rm = setupStorage.rmdir(testSuiteDir, setupCtx); waitFor(Duration.ofSeconds(10), rm); resultOf(rm); setupStorage.stop(); - setupStorage = null; elg.shutdownGracefully(); - elg = null; + allocator.close(); } } diff --git a/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageReadOnlyTest.java b/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageReadOnlyTest.java index 0e26728fc..2342d9a96 100644 --- a/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageReadOnlyTest.java +++ b/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageReadOnlyTest.java @@ -42,7 +42,7 @@ @Tag("aws-platform") public class S3StorageReadOnlyTest extends StorageReadOnlyTestSuite { - static Duration SETUP_TIMEOUT = Duration.of(5, ChronoUnit.SECONDS); + static Duration SETUP_TIMEOUT = Duration.of(10, ChronoUnit.SECONDS); static Properties storageProps; static String testSuiteDir; @@ -50,8 +50,8 @@ public class S3StorageReadOnlyTest extends StorageReadOnlyTestSuite { static EventLoopGroup elg; static BufferAllocator allocator; - static S3ObjectStorage setupStorage; - static DataContext setupCtx; + static S3ObjectStorage setupStorage, rwTestStorage, roTestStorage; + static DataContext setupCtx, testCtx; static int testNumber; @@ -75,54 +75,44 @@ static void setupStorage() throws Exception { setupStorage.start(elg); var mkdir = setupStorage.mkdir(testSuiteDir, true, setupCtx); - waitFor(Duration.ofSeconds(10), mkdir); - resultOf(mkdir); - } - - @BeforeEach - void setup() throws Exception { - - var testDir = String.format("%stest_%d", testSuiteDir, ++testNumber); - - var mkdir = setupStorage.mkdir(testDir, false, setupCtx); waitFor(SETUP_TIMEOUT, mkdir); resultOf(mkdir); - storageProps.put(S3ObjectStorage.PREFIX_PROPERTY, testDir); - rwStorage = new S3ObjectStorage("TEST_" + testNumber + "_RW", storageProps); - rwStorage.start(elg); + storageProps.put(S3ObjectStorage.PREFIX_PROPERTY, testSuiteDir); + rwTestStorage = new S3ObjectStorage("TEST_" + testNumber + "_RW", storageProps); + rwTestStorage.start(elg); var roProps = new Properties(); roProps.putAll(storageProps); roProps.put(CommonFileStorage.READ_ONLY_CONFIG_KEY, "true"); - roStorage = new S3ObjectStorage("TEST_" + testNumber + "_RO", roProps); - roStorage.start(elg); + roTestStorage = new S3ObjectStorage("TEST_" + testNumber + "_RO", roProps); + roTestStorage.start(elg); - dataContext = new DataContext(elg.next(), allocator); + testCtx = new DataContext(elg.next(), allocator); } - @AfterEach - void tearDown() { - - rwStorage.stop(); - rwStorage = null; + @BeforeEach + void setup() { - roStorage.stop(); - roStorage = null; + rwStorage = rwTestStorage; + roStorage = roTestStorage; + dataContext = testCtx; } @AfterAll static void tearDownStorage() throws Exception { + roTestStorage.stop(); + rwTestStorage.stop(); + var rm = setupStorage.rmdir(testSuiteDir, setupCtx); waitFor(Duration.ofSeconds(10), rm); resultOf(rm); setupStorage.stop(); - setupStorage = null; elg.shutdownGracefully(); - elg = null; + allocator.close(); } } diff --git a/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageReadWriteTest.java b/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageReadWriteTest.java index 07f25db28..4b6728fb9 100644 --- a/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageReadWriteTest.java +++ b/tracdap-plugins/aws-storage/src/test/java/org/finos/tracdap/plugins/aws/storage/S3StorageReadWriteTest.java @@ -42,7 +42,7 @@ @Tag("aws-platform") public class S3StorageReadWriteTest extends StorageReadWriteTestSuite { - static Duration SETUP_TIMEOUT = Duration.of(5, ChronoUnit.SECONDS); + static Duration SETUP_TIMEOUT = Duration.of(10, ChronoUnit.SECONDS); static Properties storageProps; static String testSuiteDir; @@ -50,8 +50,8 @@ public class S3StorageReadWriteTest extends StorageReadWriteTestSuite { static EventLoopGroup elg; static BufferAllocator allocator; - static S3ObjectStorage setupStorage; - static DataContext setupCtx; + static S3ObjectStorage setupStorage, testStorage; + static DataContext setupCtx, testCtx; static int testNumber; @@ -75,44 +75,35 @@ static void setupStorage() throws Exception { setupStorage.start(elg); var mkdir = setupStorage.mkdir(testSuiteDir, true, setupCtx); - waitFor(Duration.ofSeconds(10), mkdir); - resultOf(mkdir); - } - - @BeforeEach - void setup() throws Exception { - - var testDir = String.format("%stest_%d", testSuiteDir, ++testNumber); - - var mkdir = setupStorage.mkdir(testDir, false, setupCtx); waitFor(SETUP_TIMEOUT, mkdir); resultOf(mkdir); - storageProps.put(S3ObjectStorage.PREFIX_PROPERTY, testDir); - storage = new S3ObjectStorage("TEST_" + testNumber, storageProps); - storage.start(elg); + storageProps.put(S3ObjectStorage.PREFIX_PROPERTY, testSuiteDir); + testStorage = new S3ObjectStorage("TEST_" + testNumber, storageProps); + testStorage.start(elg); - dataContext = new DataContext(elg.next(), allocator); + testCtx = new DataContext(elg.next(), allocator); } - @AfterEach - void tearDown() { + @BeforeEach + void setup() { - storage.stop(); - storage = null; + storage = testStorage; + dataContext = testCtx; } @AfterAll static void tearDownStorage() throws Exception { + testStorage.stop(); + var rm = setupStorage.rmdir(testSuiteDir, setupCtx); waitFor(Duration.ofSeconds(10), rm); resultOf(rm); setupStorage.stop(); - setupStorage = null; elg.shutdownGracefully(); - elg = null; + allocator.close(); } } diff --git a/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageOperationsTest.java b/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageOperationsTest.java index f0c6c7416..1ef417bee 100644 --- a/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageOperationsTest.java +++ b/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageOperationsTest.java @@ -41,7 +41,7 @@ @Tag("azure-platform") public class AzureStorageOperationsTest extends StorageOperationsTestSuite { - static Duration SETUP_TIMEOUT = Duration.of(5, ChronoUnit.SECONDS); + static Duration SETUP_TIMEOUT = Duration.of(10, ChronoUnit.SECONDS); static Properties storageProps; static String testSuiteDir; @@ -49,8 +49,8 @@ public class AzureStorageOperationsTest extends StorageOperationsTestSuite { static EventLoopGroup elg; static BufferAllocator allocator; - static DataContext setupCtx; - static AzureBlobStorage setupStorage; + static DataContext setupCtx, testCtx; + static AzureBlobStorage setupStorage, testStorage; static int testNumber; @@ -75,42 +75,33 @@ static void setupStorage() throws Exception { var mkdir = setupStorage.mkdir(testSuiteDir, true, setupCtx); waitFor(Duration.ofSeconds(10), mkdir); resultOf(mkdir); - } - - @BeforeEach - void setup() throws Exception { - - var testDir = String.format("%stest_%d", testSuiteDir, ++testNumber); - var mkdir = setupStorage.mkdir(testDir, false, setupCtx); - waitFor(SETUP_TIMEOUT, mkdir); - resultOf(mkdir); - - storageProps.put(AzureBlobStorage.PREFIX_PROPERTY, testDir); - storage = new AzureBlobStorage("TEST_" + testNumber, storageProps); - storage.start(elg); + storageProps.put(AzureBlobStorage.PREFIX_PROPERTY, testSuiteDir); + testStorage = new AzureBlobStorage("TEST_" + testNumber, storageProps); + testStorage.start(elg); - dataContext = new DataContext(elg.next(), allocator); + testCtx = new DataContext(elg.next(), allocator); } - @AfterEach - void tearDown() { + @BeforeEach + void setup() { - storage.stop(); - storage = null; + storage = testStorage; + dataContext = testCtx; } @AfterAll static void tearDownStorage() throws Exception { + testStorage.stop(); + var rm = setupStorage.rmdir(testSuiteDir, setupCtx); waitFor(Duration.ofSeconds(10), rm); resultOf(rm); setupStorage.stop(); - setupStorage = null; elg.shutdownGracefully(); - elg = null; + allocator.close(); } } diff --git a/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageReadOnlyTest.java b/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageReadOnlyTest.java index a04ff6852..7d75a5f22 100644 --- a/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageReadOnlyTest.java +++ b/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageReadOnlyTest.java @@ -42,7 +42,7 @@ @Tag("azure-platform") public class AzureStorageReadOnlyTest extends StorageReadOnlyTestSuite { - static Duration SETUP_TIMEOUT = Duration.of(5, ChronoUnit.SECONDS); + static Duration SETUP_TIMEOUT = Duration.of(10, ChronoUnit.SECONDS); static Properties storageProps; static String testSuiteDir; @@ -50,8 +50,8 @@ public class AzureStorageReadOnlyTest extends StorageReadOnlyTestSuite { static EventLoopGroup elg; static BufferAllocator allocator; - static DataContext setupCtx; - static AzureBlobStorage setupStorage; + static DataContext setupCtx, testCtx; + static AzureBlobStorage setupStorage, testRwStorage, testRoStorage; static int testNumber; @@ -74,54 +74,44 @@ static void setupStorage() throws Exception { setupStorage.start(elg); var mkdir = setupStorage.mkdir(testSuiteDir, true, setupCtx); - waitFor(Duration.ofSeconds(10), mkdir); - resultOf(mkdir); - } - - @BeforeEach - void setup() throws Exception { - - var testDir = String.format("%stest_%d", testSuiteDir, ++testNumber); - - var mkdir = setupStorage.mkdir(testDir, false, setupCtx); waitFor(SETUP_TIMEOUT, mkdir); resultOf(mkdir); - storageProps.put(AzureBlobStorage.PREFIX_PROPERTY, testDir); - rwStorage = new AzureBlobStorage("TEST_" + testNumber + "_RW", storageProps); - rwStorage.start(elg); + storageProps.put(AzureBlobStorage.PREFIX_PROPERTY, testSuiteDir); + testRwStorage = new AzureBlobStorage("TEST_" + testNumber + "_RW", storageProps); + testRwStorage.start(elg); var roProps = new Properties(); roProps.putAll(storageProps); roProps.put(CommonFileStorage.READ_ONLY_CONFIG_KEY, "true"); - roStorage = new AzureBlobStorage("TEST_" + testNumber + "_RO", roProps); - roStorage.start(elg); + testRoStorage = new AzureBlobStorage("TEST_" + testNumber + "_RO", roProps); + testRoStorage.start(elg); - dataContext = new DataContext(elg.next(), allocator); + testCtx = new DataContext(elg.next(), allocator); } - @AfterEach - void tearDown() { - - rwStorage.stop(); - rwStorage = null; + @BeforeEach + void setup() { - roStorage.stop(); - roStorage = null; + rwStorage = testRwStorage; + roStorage = testRoStorage; + dataContext = testCtx; } @AfterAll static void tearDownStorage() throws Exception { + testRwStorage.stop(); + testRoStorage.stop(); + var rm = setupStorage.rmdir(testSuiteDir, setupCtx); waitFor(Duration.ofSeconds(10), rm); resultOf(rm); setupStorage.stop(); - setupStorage = null; elg.shutdownGracefully(); - elg = null; + allocator.close(); } } diff --git a/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageReadWriteTest.java b/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageReadWriteTest.java index f1a0c4887..8fb839cc9 100644 --- a/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageReadWriteTest.java +++ b/tracdap-plugins/azure-storage/src/test/java/org/finos/tracdap/plugins/azure/storage/AzureStorageReadWriteTest.java @@ -41,7 +41,7 @@ @Tag("azure-platform") public class AzureStorageReadWriteTest extends StorageReadWriteTestSuite { - static Duration SETUP_TIMEOUT = Duration.of(5, ChronoUnit.SECONDS); + static Duration SETUP_TIMEOUT = Duration.of(10, ChronoUnit.SECONDS); static Properties storageProps; static String testSuiteDir; @@ -49,8 +49,8 @@ public class AzureStorageReadWriteTest extends StorageReadWriteTestSuite { static EventLoopGroup elg; static BufferAllocator allocator; - static DataContext setupCtx; - static AzureBlobStorage setupStorage; + static DataContext setupCtx, testCtx; + static AzureBlobStorage setupStorage, testStorage; static int testNumber; @@ -73,44 +73,35 @@ static void setupStorage() throws Exception { setupStorage.start(elg); var mkdir = setupStorage.mkdir(testSuiteDir, true, setupCtx); - waitFor(Duration.ofSeconds(10), mkdir); - resultOf(mkdir); - } - - @BeforeEach - void setup() throws Exception { - - var testDir = String.format("%stest_%d", testSuiteDir, ++testNumber); - - var mkdir = setupStorage.mkdir(testDir, false, setupCtx); waitFor(SETUP_TIMEOUT, mkdir); resultOf(mkdir); - storageProps.put(AzureBlobStorage.PREFIX_PROPERTY, testDir); - storage = new AzureBlobStorage("TEST_" + testNumber, storageProps); - storage.start(elg); + storageProps.put(AzureBlobStorage.PREFIX_PROPERTY, testSuiteDir); + testStorage = new AzureBlobStorage("TEST_" + testNumber, storageProps); + testStorage.start(elg); - dataContext = new DataContext(elg.next(), allocator); + testCtx = new DataContext(elg.next(), allocator); } - @AfterEach - void tearDown() { + @BeforeEach + void setup() { - storage.stop(); - storage = null; + storage = testStorage; + dataContext = testCtx; } @AfterAll static void tearDownStorage() throws Exception { + testStorage.stop(); + var rm = setupStorage.rmdir(testSuiteDir, setupCtx); waitFor(Duration.ofSeconds(10), rm); resultOf(rm); setupStorage.stop(); - setupStorage = null; elg.shutdownGracefully(); - elg = null; + allocator.close(); } } \ No newline at end of file diff --git a/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcpStorageReadOnlyTest.java b/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcpStorageReadOnlyTest.java index 06095596f..344e5ab77 100644 --- a/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcpStorageReadOnlyTest.java +++ b/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcpStorageReadOnlyTest.java @@ -41,7 +41,7 @@ @Tag("gcp-platform") public class GcpStorageReadOnlyTest extends StorageReadOnlyTestSuite { - static Duration SETUP_TIMEOUT = Duration.of(5, ChronoUnit.SECONDS); + static Duration SETUP_TIMEOUT = Duration.of(10, ChronoUnit.SECONDS); static Properties storageProps; static String testSuiteDir; @@ -49,18 +49,17 @@ public class GcpStorageReadOnlyTest extends StorageReadOnlyTestSuite { static EventLoopGroup elg; static BufferAllocator allocator; - static DataContext setupCtx; - static GcsObjectStorage setupStorage; - - static int testNumber; + static DataContext setupCtx, testCtx; + static GcsObjectStorage setupStorage, testRwStorage, testRoStorage; @BeforeAll static void setupStorage() throws Exception { + storageProps = GcsStorageEnvProps.readStorageEnvProps(); + var timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now()).replace(':', '.'); var random = new Random().nextLong(); - storageProps = GcsStorageEnvProps.readStorageEnvProps(); testSuiteDir = String.format( "platform_storage_ops_test_suite_%s_0x%h/", timestamp, random); @@ -73,55 +72,44 @@ static void setupStorage() throws Exception { setupStorage.start(elg); var mkdir = setupStorage.mkdir(testSuiteDir, true, setupCtx); - waitFor(Duration.ofSeconds(10), mkdir); - resultOf(mkdir); - } - - @BeforeEach - void setup() throws Exception { - - var testDir = String.format("%stest_%d", testSuiteDir, ++testNumber); - - var mkdir = setupStorage.mkdir(testDir, false, setupCtx); waitFor(SETUP_TIMEOUT, mkdir); resultOf(mkdir); - storageProps.put(GcsObjectStorage.PREFIX_PROPERTY, testDir); - rwStorage = new GcsObjectStorage("TEST_" + testNumber + "_RW", storageProps); - rwStorage.start(elg); + storageProps.put(GcsObjectStorage.PREFIX_PROPERTY, testSuiteDir); var roProps = new Properties(); roProps.putAll(storageProps); roProps.put(CommonFileStorage.READ_ONLY_CONFIG_KEY, "true"); - roStorage = new GcsObjectStorage("TEST_" + testNumber + "_RO", roProps); - roStorage.start(elg); - - dataContext = new DataContext(elg.next(), allocator); + testCtx = new DataContext(elg.next(), allocator); + testRwStorage = new GcsObjectStorage("TEST_STORAGE_RW", storageProps); + testRwStorage.start(elg); + testRoStorage = new GcsObjectStorage("TEST_STORAGE_RO", roProps); + testRoStorage.start(elg); } - @AfterEach - void tearDown() { - - rwStorage.stop(); - rwStorage = null; + @BeforeEach + void setup() { - roStorage.stop(); - roStorage = null; + dataContext = testCtx; + rwStorage = testRwStorage; + roStorage = testRoStorage; } @AfterAll static void tearDownStorage() throws Exception { + testRoStorage.stop(); + testRwStorage.stop(); + var rm = setupStorage.rmdir(testSuiteDir, setupCtx); waitFor(Duration.ofSeconds(10), rm); resultOf(rm); setupStorage.stop(); - setupStorage = null; elg.shutdownGracefully(); - elg = null; + allocator.close(); } } diff --git a/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcpStorageReadWriteTest.java b/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcpStorageReadWriteTest.java index b5e788ffd..3b921905a 100644 --- a/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcpStorageReadWriteTest.java +++ b/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcpStorageReadWriteTest.java @@ -41,7 +41,7 @@ @Tag("gcp-platform") public class GcpStorageReadWriteTest extends StorageReadWriteTestSuite { - static Duration SETUP_TIMEOUT = Duration.of(5, ChronoUnit.SECONDS); + static Duration SETUP_TIMEOUT = Duration.of(10, ChronoUnit.SECONDS); static Properties storageProps; static String testSuiteDir; @@ -49,68 +49,58 @@ public class GcpStorageReadWriteTest extends StorageReadWriteTestSuite { static EventLoopGroup elg; static BufferAllocator allocator; - static DataContext setupCtx; - static GcsObjectStorage setupStorage; - - static int testNumber; + static DataContext setupCtx, testCtx; + static GcsObjectStorage setupStorage, testStorage; @BeforeAll static void setupStorage() throws Exception { + storageProps = GcsStorageEnvProps.readStorageEnvProps(); + var timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now()).replace(':', '.'); var random = new Random().nextLong(); - storageProps = GcsStorageEnvProps.readStorageEnvProps(); testSuiteDir = String.format( - "platform_storage_ops_test_suite_%s_0x%h/", + "platform_storage_readwrite_test_suite_%s_0x%h/", timestamp, random); elg = new NioEventLoopGroup(2, new DefaultThreadFactory("ops-test")); allocator = new RootAllocator(); setupCtx = new DataContext(elg.next(), allocator); - setupStorage = new GcsObjectStorage("STORAGE_SETUP", storageProps); + setupStorage = new GcsObjectStorage("SETUP_STORAGE", storageProps); setupStorage.start(elg); var mkdir = setupStorage.mkdir(testSuiteDir, true, setupCtx); - waitFor(Duration.ofSeconds(10), mkdir); - resultOf(mkdir); - } - - @BeforeEach - void setup() throws Exception { - - var testDir = String.format("%stest_%d", testSuiteDir, ++testNumber); - - var mkdir = setupStorage.mkdir(testDir, false, setupCtx); waitFor(SETUP_TIMEOUT, mkdir); resultOf(mkdir); - storageProps.put(GcsObjectStorage.PREFIX_PROPERTY, testDir); - storage = new GcsObjectStorage("TEST_" + testNumber, storageProps); - storage.start(elg); + storageProps.put(GcsObjectStorage.PREFIX_PROPERTY, testSuiteDir); - dataContext = new DataContext(elg.next(), allocator); + testCtx = new DataContext(elg.next(), allocator); + testStorage = new GcsObjectStorage("TEST_STORAGE", storageProps); + testStorage.start(elg); } - @AfterEach - void tearDown() { + @BeforeEach + void setup() { - storage.stop(); - storage = null; + dataContext = testCtx; + storage = testStorage; } @AfterAll static void tearDownStorage() throws Exception { + testStorage.stop(); + var rm = setupStorage.rmdir(testSuiteDir, setupCtx); waitFor(Duration.ofSeconds(10), rm); resultOf(rm); setupStorage.stop(); - setupStorage = null; elg.shutdownGracefully(); - elg = null; + allocator.close(); } } diff --git a/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcsStorageOperationsTest.java b/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcsStorageOperationsTest.java index 11722337a..6346c2ce5 100644 --- a/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcsStorageOperationsTest.java +++ b/tracdap-plugins/gcp-storage/src/test/java/org/finos/tracdap/plugins/gcp/storage/GcsStorageOperationsTest.java @@ -43,7 +43,7 @@ @Tag("gcp-platform") public class GcsStorageOperationsTest extends StorageOperationsTestSuite { - static Duration SETUP_TIMEOUT = Duration.of(5, ChronoUnit.SECONDS); + static Duration SETUP_TIMEOUT = Duration.of(10, ChronoUnit.SECONDS); static Properties storageProps; static String testSuiteDir; @@ -51,18 +51,17 @@ public class GcsStorageOperationsTest extends StorageOperationsTestSuite { static EventLoopGroup elg; static BufferAllocator allocator; - static DataContext setupCtx; - static GcsObjectStorage setupStorage; - - static int testNumber; + static DataContext setupCtx, testCtx; + static GcsObjectStorage setupStorage, testStorage; @BeforeAll static void setupStorage() throws Exception { + storageProps = GcsStorageEnvProps.readStorageEnvProps(); + var timestamp = DateTimeFormatter.ISO_INSTANT.format(Instant.now()).replace(':', '.'); var random = new Random().nextLong(); - storageProps = GcsStorageEnvProps.readStorageEnvProps(); testSuiteDir = String.format( "platform_storage_ops_test_suite_%s_0x%h/", timestamp, random); @@ -71,46 +70,37 @@ static void setupStorage() throws Exception { allocator = new RootAllocator(); setupCtx = new DataContext(elg.next(), allocator); - setupStorage = new GcsObjectStorage("STORAGE_SETUP", storageProps); + setupStorage = new GcsObjectStorage("SETUP_STORAGE", storageProps); setupStorage.start(elg); var mkdir = setupStorage.mkdir(testSuiteDir, true, setupCtx); - waitFor(Duration.ofSeconds(10), mkdir); - resultOf(mkdir); - } - - @BeforeEach - void setup() throws Exception { - - var testDir = String.format("%stest_%d", testSuiteDir, ++testNumber); - - var mkdir = setupStorage.mkdir(testDir, false, setupCtx); waitFor(SETUP_TIMEOUT, mkdir); resultOf(mkdir); - storageProps.put(GcsObjectStorage.PREFIX_PROPERTY, testDir); - storage = new GcsObjectStorage("TEST_" + testNumber, storageProps); - storage.start(elg); + storageProps.put(GcsObjectStorage.PREFIX_PROPERTY, testSuiteDir); - dataContext = new DataContext(elg.next(), allocator); + testCtx = new DataContext(elg.next(), allocator); + testStorage = new GcsObjectStorage("TEST_STORAGE", storageProps); + testStorage.start(elg); } - @AfterEach - void tearDown() { + @BeforeEach + void setup() { - storage.stop(); - storage = null; + dataContext = testCtx; + storage = testStorage; } @AfterAll static void tearDownStorage() throws Exception { + testStorage.stop(); + var rm = setupStorage.rmdir(testSuiteDir, setupCtx); waitFor(Duration.ofSeconds(10), rm); resultOf(rm); setupStorage.stop(); - setupStorage = null; elg.shutdownGracefully(); elg = null; diff --git a/tracdap-runtime/python/test/tracdap_test/rt/impl/test_storage_local.py b/tracdap-runtime/python/test/tracdap_test/rt/impl/test_storage_local.py index 812db06bb..14b73c931 100644 --- a/tracdap-runtime/python/test/tracdap_test/rt/impl/test_storage_local.py +++ b/tracdap-runtime/python/test/tracdap_test/rt/impl/test_storage_local.py @@ -37,25 +37,16 @@ class LocalArrowImplStorageTest(unittest.TestCase, FileOperationsTestSuite, FileReadWriteTestSuite): storage_root: tempfile.TemporaryDirectory - test_number: int @classmethod def setUpClass(cls) -> None: cls.storage_root = tempfile.TemporaryDirectory() - cls.test_number = 0 - - def setUp(self): - - test_dir = pathlib.Path(self.storage_root.name).joinpath(f"test_{self.test_number}") - test_dir.mkdir() - - self.__class__.test_number += 1 test_storage_config = _cfg.PluginConfig( protocol="LOCAL", properties={ - "rootPath": str(test_dir), + "rootPath": cls.storage_root.name, "runtimeFs": "arrow"}) sys_config = _cfg.RuntimeConfig() @@ -63,7 +54,8 @@ def setUp(self): sys_config.storage.buckets["test_bucket"] = test_storage_config manager = _storage.StorageManager(sys_config) - self.storage = manager.get_file_storage("test_bucket") + + cls.storage = manager.get_file_storage("test_bucket") @classmethod def tearDownClass(cls) -> None: @@ -75,25 +67,16 @@ def tearDownClass(cls) -> None: class LocalPythonImplStorageTest(unittest.TestCase, FileOperationsTestSuite, FileReadWriteTestSuite): storage_root: tempfile.TemporaryDirectory - test_number: int @classmethod def setUpClass(cls) -> None: cls.storage_root = tempfile.TemporaryDirectory() - cls.test_number = 0 - - def setUp(self): - - test_dir = pathlib.Path(self.storage_root.name).joinpath(f"test_{self.test_number}") - test_dir.mkdir() - - self.__class__.test_number += 1 test_storage_config = _cfg.PluginConfig( protocol="LOCAL", properties={ - "rootPath": str(test_dir), + "rootPath": cls.storage_root.name, "runtimeFs": "python"}) sys_config = _cfg.RuntimeConfig() @@ -101,7 +84,7 @@ def setUp(self): sys_config.storage.buckets["test_bucket"] = test_storage_config manager = _storage.StorageManager(sys_config) - self.storage = manager.get_file_storage("test_bucket") + cls.storage = manager.get_file_storage("test_bucket") @classmethod def tearDownClass(cls) -> None: diff --git a/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_aws.py b/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_aws.py index 890094075..d46863838 100644 --- a/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_aws.py +++ b/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_aws.py @@ -31,35 +31,22 @@ class S3ArrowStorageTest(unittest.TestCase, FileOperationsTestSuite, FileReadWri suite_storage_prefix = f"runtime_storage_test_suite_{uuid.uuid4()}" suite_storage: storage.IFileStorage - test_number: int @classmethod def setUpClass(cls) -> None: - properties = cls._properties_from_env() + suite_properties = cls._properties_from_env() + suite_storage_config = cfg.PluginConfig(protocol="S3", properties=suite_properties) - suite_storage_config = cfg.PluginConfig(protocol="S3", properties=properties) - - cls.suite_storage = cls._storage_from_config(suite_storage_config, "tracdap_ci_storage") + cls.suite_storage = cls._storage_from_config(suite_storage_config, "tracdap_ci_storage_setup") cls.suite_storage.mkdir(cls.suite_storage_prefix) - cls.test_number = 0 - - def setUp(self): - - test_name = f"test_aws_{self.test_number}" - test_dir = f"{self.suite_storage_prefix}/{test_name}" - - self.suite_storage.mkdir(test_dir) - - self.__class__.test_number += 1 - - properties = self._properties_from_env() - properties["arrowNativeFs"] = "true" - properties["prefix"] = test_dir - test_storage_config = cfg.PluginConfig(protocol="S3", properties=properties) + test_properties = cls._properties_from_env() + test_properties["arrowNativeFs"] = "true" + test_properties["prefix"] = cls.suite_storage_prefix + test_storage_config = cfg.PluginConfig(protocol="S3", properties=test_properties) - self.storage = self._storage_from_config(test_storage_config, test_name) + cls.storage = cls._storage_from_config(test_storage_config, "tracdap_ci_storage") @classmethod def tearDownClass(cls) -> None: diff --git a/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_azure.py b/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_azure.py index b46e8e5f7..a2789e0a3 100644 --- a/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_azure.py +++ b/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_azure.py @@ -30,34 +30,21 @@ class BlobFsspecStorageTest(unittest.TestCase, FileOperationsTestSuite, FileRead suite_storage_prefix = f"runtime_storage_test_suite_{uuid.uuid4()}" suite_storage: storage.IFileStorage - test_number: int @classmethod def setUpClass(cls) -> None: - properties = cls._properties_from_env() + suite_properties = cls._properties_from_env() + suite_storage_config = cfg.PluginConfig(protocol="BLOB", properties=suite_properties) - suite_storage_config = cfg.PluginConfig(protocol="BLOB", properties=properties) - - cls.suite_storage = cls._storage_from_config(suite_storage_config, "tracdap_ci_storage") + cls.suite_storage = cls._storage_from_config(suite_storage_config, "tracdap_ci_storage_setup") cls.suite_storage.mkdir(cls.suite_storage_prefix) - cls.test_number = 0 - - def setUp(self): - - test_name = f"test_azure_{self.test_number}" - test_dir = f"{self.suite_storage_prefix}/{test_name}" - - self.suite_storage.mkdir(test_dir) - - self.__class__.test_number += 1 - - properties = self._properties_from_env() - properties["prefix"] = test_dir - test_storage_config = cfg.PluginConfig(protocol="BLOB", properties=properties) + test_properties = cls._properties_from_env() + test_properties["prefix"] = cls.suite_storage_prefix + test_storage_config = cfg.PluginConfig(protocol="BLOB", properties=test_properties) - self.storage = self._storage_from_config(test_storage_config, test_name) + cls.storage = cls._storage_from_config(test_storage_config, "tracdap_ci_storage") @classmethod def tearDownClass(cls) -> None: diff --git a/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_gcp.py b/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_gcp.py index 77da9ab19..6bd0c4f9c 100644 --- a/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_gcp.py +++ b/tracdap-runtime/python/test/tracdap_test/rt/plugins/int_storage_gcp.py @@ -38,34 +38,21 @@ class GcsArrowStorageTest(unittest.TestCase, FileOperationsTestSuite, FileReadWr suite_storage_prefix = f"runtime_storage_test_suite_{uuid.uuid4()}" suite_storage: storage.IFileStorage - test_number: int @classmethod def setUpClass(cls) -> None: - properties = cls._properties_from_env() + suite_properties = cls._properties_from_env() + suite_storage_config = cfg.PluginConfig(protocol="GCS", properties=suite_properties) - suite_storage_config = cfg.PluginConfig(protocol="GCS", properties=properties) - - cls.suite_storage = cls._storage_from_config(suite_storage_config, "tracdap_ci_storage") + cls.suite_storage = cls._storage_from_config(suite_storage_config, "tracdap_ci_storage_setup") cls.suite_storage.mkdir(cls.suite_storage_prefix) - cls.test_number = 0 - - def setUp(self): - - test_name = f"test_gcp_{self.test_number}" - test_dir = f"{self.suite_storage_prefix}/{test_name}" - - self.suite_storage.mkdir(test_dir) - - self.__class__.test_number += 1 - - properties = self._properties_from_env() - properties["prefix"] = test_dir - test_storage_config = cfg.PluginConfig(protocol="GCS", properties=properties) + test_properties = cls._properties_from_env() + test_properties["prefix"] = cls.suite_storage_prefix + test_storage_config = cfg.PluginConfig(protocol="GCS", properties=test_properties) - self.storage = self._storage_from_config(test_storage_config, test_name) + cls.storage = cls._storage_from_config(test_storage_config, "tracdap_ci_storage") @classmethod def tearDownClass(cls) -> None: @@ -106,34 +93,21 @@ class GcsFsspecStorageTest(unittest.TestCase, FileOperationsTestSuite, FileReadW suite_storage_prefix = f"runtime_storage_test_suite_{uuid.uuid4()}" suite_storage: storage.IFileStorage - test_number: int @classmethod def setUpClass(cls) -> None: - properties = cls._properties_from_env() + suite_properties = cls._properties_from_env() + suite_config = cfg.PluginConfig(protocol="GCS", properties=suite_properties) - suite_storage_config = cfg.PluginConfig(protocol="GCS", properties=properties) - - cls.suite_storage = cls._storage_from_config(suite_storage_config, "tracdap_ci_storage") + cls.suite_storage = cls._storage_from_config(suite_config, "tracdap_ci_storage_setup") cls.suite_storage.mkdir(cls.suite_storage_prefix) - cls.test_number = 0 - - def setUp(self): - - test_name = f"test_gcp_{self.test_number}" - test_dir = f"{self.suite_storage_prefix}/{test_name}" - - self.suite_storage.mkdir(test_dir) - - self.__class__.test_number += 1 - - properties = self._properties_from_env() - properties["prefix"] = test_dir - test_storage_config = cfg.PluginConfig(protocol="GCS", properties=properties) + test_properties = cls._properties_from_env() + test_properties["prefix"] = cls.suite_storage_prefix + test_config = cfg.PluginConfig(protocol="GCS", properties=test_properties) - self.storage = self._storage_from_config(test_storage_config, test_name) + cls.storage = cls._storage_from_config(test_config, "tracdap_ci_storage") @classmethod def tearDownClass(cls) -> None: diff --git a/tracdap-runtime/python/test/tracdap_test/rt/suites/file_storage_suite.py b/tracdap-runtime/python/test/tracdap_test/rt/suites/file_storage_suite.py index 4e44abe5a..3e5cb334d 100644 --- a/tracdap-runtime/python/test/tracdap_test/rt/suites/file_storage_suite.py +++ b/tracdap-runtime/python/test/tracdap_test/rt/suites/file_storage_suite.py @@ -56,8 +56,14 @@ class FileOperationsTestSuite: skipTest = unittest.TestCase.skipTest # Windows limits individual path segments to 255 chars - LONG_PATH_TXT_FILE = "long_" + "A" * 246 + ".txt" - LONG_PATH_DIR = "long_" + "A" * 250 + + @classmethod + def make_long_path(cls, prefix, suffix): + return prefix + "A" * (255 - len(prefix) - len(suffix)) + suffix + + @classmethod + def make_long_dir_path(cls, prefix): + return prefix + "A" * (255 - len(prefix)) def __init__(self): self.storage: _storage.IFileStorage = None # noqa @@ -68,48 +74,48 @@ def __init__(self): def test_exists_file(self): - self.make_small_file("test_file.txt") + self.make_small_file("test_exists_file.txt") - file_present = self.storage.exists("test_file.txt") - file_not_present = self.storage.exists("other_file.txt") + file_present = self.storage.exists("test_exists_file.txt") + file_not_present = self.storage.exists("test_exists_file_other.txt") self.assertTrue(file_present) self.assertFalse(file_not_present) def test_exists_long_path(self): - self.make_small_file(self.LONG_PATH_TXT_FILE) + long_path = self.make_long_path("test_exists_long_path", ".txt") + + self.make_small_file(long_path) - file_present = self.storage.exists(self.LONG_PATH_TXT_FILE) - file_not_present = self.storage.exists("other_file.txt") + file_present = self.storage.exists(long_path) self.assertTrue(file_present) - self.assertFalse(file_not_present) def test_exists_empty_file(self): - self.make_file("test_file.txt", b"") + self.make_file("test_exists_empty_file.txt", b"") - empty_file_exist = self.storage.exists("test_file.txt") + empty_file_exist = self.storage.exists("test_exists_empty_file.txt") self.assertTrue(empty_file_exist) def test_exists_dir(self): - self.storage.mkdir("test_dir", False) + self.storage.mkdir("test_exists_dir", False) - dir_present = self.storage.exists("test_dir") - dir_not_present = self.storage.exists("other_dir") + dir_present = self.storage.exists("test_exists_dir") + dir_not_present = self.storage.exists("test_exists_dir_other") self.assertTrue(dir_present) self.assertFalse(dir_not_present) def test_exists_parent_dir(self): - self.storage.mkdir("parent_dir/child_dir", True) + self.storage.mkdir("test_exists_parent_dir/child_dir", True) - dir_present = self.storage.exists("parent_dir") - dir_not_present = self.storage.exists("other_dir") + dir_present = self.storage.exists("test_exists_parent_dir") + dir_not_present = self.storage.exists("test_exists_parent_dir_other") self.assertTrue(dir_present) self.assertFalse(dir_not_present) @@ -135,40 +141,42 @@ def test_size_ok(self): content = "Content of a certain size\n".encode('utf-8') expected_size = len(content) - self.make_file("test_file.txt", content) + self.make_file("test_size_ok.txt", content) - size = self.storage.size("test_file.txt") + size = self.storage.size("test_size_ok.txt") self.assertEqual(expected_size, size) def test_size_long_path(self): + long_path = self.make_long_path("test_size_long_path", ".txt") + content = "Content of a certain size\n".encode('utf-8') expected_size = len(content) - self.make_file(self.LONG_PATH_TXT_FILE, content) + self.make_file(long_path, content) - size = self.storage.size(self.LONG_PATH_TXT_FILE) + size = self.storage.size(long_path) self.assertEqual(expected_size, size) def test_size_empty_file(self): - self.make_file("test_file.txt", b"") + self.make_file("test_size_empty_file.txt", b"") - size = self.storage.size("test_file.txt") + size = self.storage.size("test_size_empty_file.txt") self.assertEqual(0, size) def test_size_dir(self): - self.storage.mkdir("test_dir", False) + self.storage.mkdir("test_size_dir", False) - self.assertRaises(_ex.EStorageRequest, lambda: self.storage.size("test_dir")) + self.assertRaises(_ex.EStorageRequest, lambda: self.storage.size("test_size_dir")) def test_size_missing(self): - self.assertRaises(_ex.EStorageRequest, lambda: self.storage.size("missing_file.txt")) + self.assertRaises(_ex.EStorageRequest, lambda: self.storage.size("test_size_missing.txt")) def test_size_storage_root(self): @@ -191,12 +199,12 @@ def test_stat_file_ok(self): content = "Sample content for stat call\n".encode('utf-8') expected_size = len(content) - self.storage.mkdir("some_dir", False) - self.make_file("some_dir/test_file.txt", content) + self.storage.mkdir("test_stat_file_ok", False) + self.make_file("test_stat_file_ok/test_file.txt", content) - stat_result = self.storage.stat("some_dir/test_file.txt") + stat_result = self.storage.stat("test_stat_file_ok/test_file.txt") - self.assertEqual("some_dir/test_file.txt", stat_result.storage_path) + self.assertEqual("test_stat_file_ok/test_file.txt", stat_result.storage_path) self.assertEqual("test_file.txt", stat_result.file_name) self.assertEqual(_storage.FileType.FILE, stat_result.file_type) self.assertEqual(expected_size, stat_result.size) @@ -205,16 +213,18 @@ def test_stat_file_long_path(self): # Simple case - stat a file + long_path = self.make_long_path("test_stat_file_long_path", ".txt") + content = "Sample content for stat call\n".encode('utf-8') expected_size = len(content) - self.storage.mkdir("some_dir", False) - self.make_file("some_dir/" + self.LONG_PATH_TXT_FILE, content) + self.storage.mkdir("test_stat_file_long_path", False) + self.make_file("test_stat_file_long_path/" + long_path, content) - stat_result = self.storage.stat("some_dir/" + self.LONG_PATH_TXT_FILE,) + stat_result = self.storage.stat("test_stat_file_long_path/" + long_path,) - self.assertEqual("some_dir/" + self.LONG_PATH_TXT_FILE, stat_result.storage_path) - self.assertEqual(self.LONG_PATH_TXT_FILE, stat_result.file_name) + self.assertEqual("test_stat_file_long_path/" + long_path, stat_result.storage_path) + self.assertEqual(long_path, stat_result.file_name) self.assertEqual(_storage.FileType.FILE, stat_result.file_type) self.assertEqual(expected_size, stat_result.size) @@ -226,9 +236,9 @@ def test_stat_file_mtime(self): test_start = dt.datetime.now(dt.timezone.utc) time.sleep(1.0) # Let time elapse before/after the test calls - self.make_small_file("test_file.txt") + self.make_small_file("test_stat_file_mtime.txt") - stat_result = self.storage.stat("test_file.txt") + stat_result = self.storage.stat("test_stat_file_mtime.txt") time.sleep(1.0) # Let time elapse before/after the test calls test_finish = dt.datetime.now(dt.timezone.utc) @@ -253,14 +263,14 @@ def test_stat_file_atime(self): # On FAT32, atime is limited to an access date, i.e. one-day resolution - self.make_small_file("test_file.txt") + self.make_small_file("test_stat_file_atime.txt") test_start = dt.datetime.now(dt.timezone.utc) time.sleep(0.01) # Let time elapse before/after the test calls - self.storage.read_bytes("test_file.txt") + self.storage.read_bytes("test_stat_file_atime.txt") - stat_result = self.storage.stat("test_file.txt") + stat_result = self.storage.stat("test_stat_file_atime.txt") time.sleep(0.01) # Let time elapse before/after the test calls test_finish = dt.datetime.now(dt.timezone.utc) @@ -270,11 +280,11 @@ def test_stat_file_atime(self): def test_stat_dir_ok(self): - self.storage.mkdir("some_dir/test_dir", True) + self.storage.mkdir("test_stat_dir_ok/test_dir", True) - stat_result = self.storage.stat("some_dir/test_dir") + stat_result = self.storage.stat("test_stat_dir_ok/test_dir") - self.assertEqual("some_dir/test_dir", stat_result.storage_path) + self.assertEqual("test_stat_dir_ok/test_dir", stat_result.storage_path) self.assertEqual("test_dir", stat_result.file_name) self.assertEqual(_storage.FileType.DIRECTORY, stat_result.file_type) @@ -283,12 +293,14 @@ def test_stat_dir_ok(self): def test_stat_dir_long_path(self): - self.storage.mkdir("some_dir/" + self.LONG_PATH_DIR, True) + long_path = self.make_long_dir_path("test_stat_dir_long_path") - stat_result = self.storage.stat("some_dir/" + self.LONG_PATH_DIR,) + self.storage.mkdir("test_stat_dir_long_path/" + long_path, True) - self.assertEqual("some_dir/" + self.LONG_PATH_DIR, stat_result.storage_path) - self.assertEqual(self.LONG_PATH_DIR, stat_result.file_name) + stat_result = self.storage.stat("test_stat_dir_long_path/" + long_path,) + + self.assertEqual("test_stat_dir_long_path/" + long_path, stat_result.storage_path) + self.assertEqual(long_path, stat_result.file_name) self.assertEqual(_storage.FileType.DIRECTORY, stat_result.file_type) # Size field for directories should always be set to 0 @@ -296,12 +308,12 @@ def test_stat_dir_long_path(self): def test_stat_dir_implicit_ok(self): - self.storage.mkdir("some_dir/test_dir", True) + self.storage.mkdir("test_stat_dir_implicit_ok/test_dir", True) - stat_result = self.storage.stat("some_dir") + stat_result = self.storage.stat("test_stat_dir_implicit_ok") - self.assertEqual("some_dir", stat_result.storage_path) - self.assertEqual("some_dir", stat_result.file_name) + self.assertEqual("test_stat_dir_implicit_ok", stat_result.storage_path) + self.assertEqual("test_stat_dir_implicit_ok", stat_result.file_name) self.assertEqual(_storage.FileType.DIRECTORY, stat_result.file_type) # Size field for directories should always be set to 0 @@ -312,16 +324,16 @@ def test_stat_dir_mtime(self): # mtime and atime for dirs is unlikely to be supported in cloud storage buckets # So, all of these fields are optional in stat responses for directories - self.storage.mkdir("some_dir/test_dir", True) + self.storage.mkdir("test_stat_dir_mtime/test_dir", True) # "Modify" the directory by adding a file to it test_start = dt.datetime.now(dt.timezone.utc) time.sleep(0.01) # Let time elapse before/after the test calls - self.make_small_file("some_dir/test_dir/a_file.txt") + self.make_small_file("test_stat_dir_mtime/test_dir/a_file.txt") - stat_result = self.storage.stat("some_dir/test_dir") + stat_result = self.storage.stat("test_stat_dir_mtime/test_dir") time.sleep(0.01) # Let time elapse before/after the test calls test_finish = dt.datetime.now(dt.timezone.utc) @@ -338,17 +350,17 @@ def test_stat_dir_atime(self): # mtime and atime for dirs is unlikely to be supported in cloud storage buckets # So, all of these fields are optional in stat responses for directories - self.storage.mkdir("some_dir/test_dir", True) - self.make_small_file("some_dir/test_dir/a_file.txt") + self.storage.mkdir("test_stat_dir_atime/test_dir", True) + self.make_small_file("test_stat_dir_atime/test_dir/a_file.txt") # Access the directory by running "ls" on it test_start = dt.datetime.now(dt.timezone.utc) time.sleep(0.01) # Let time elapse before/after the test calls - self.storage.ls("some_dir/test_dir") + self.storage.ls("test_stat_dir_atime/test_dir") - stat_result = self.storage.stat("some_dir/test_dir") + stat_result = self.storage.stat("test_stat_dir_atime/test_dir") time.sleep(0.01) # Let time elapse before/after the test calls test_finish = dt.datetime.now(dt.timezone.utc) @@ -383,11 +395,11 @@ def test_ls_ok(self): # Simple listing, dir containing one file and one sub dir - self.storage.mkdir("test_dir", False) - self.storage.mkdir("test_dir/child_1", False) - self.make_small_file("test_dir/child_2.txt") + self.storage.mkdir("test_ls_ok", False) + self.storage.mkdir("test_ls_ok/child_1", False) + self.make_small_file("test_ls_ok/child_2.txt") - ls = self.storage.ls("test_dir") + ls = self.storage.ls("test_ls_ok") self.assertEqual(2, len(ls)) @@ -395,22 +407,24 @@ def test_ls_ok(self): child2 = next(filter(lambda x: x.file_name == "child_2.txt", ls), None) self.assertIsNotNone(child1) - self.assertEqual("test_dir/child_1", child1.storage_path) + self.assertEqual("test_ls_ok/child_1", child1.storage_path) self.assertEqual(_storage.FileType.DIRECTORY, child1.file_type) self.assertIsNotNone(child2) - self.assertEqual("test_dir/child_2.txt", child2.storage_path) + self.assertEqual("test_ls_ok/child_2.txt", child2.storage_path) self.assertEqual(_storage.FileType.FILE, child2.file_type) def test_ls_long_path(self): # Simple listing, dir containing one file and one sub dir - self.storage.mkdir(self.LONG_PATH_DIR, False) - self.storage.mkdir(self.LONG_PATH_DIR + "/child_1", False) - self.make_small_file(self.LONG_PATH_DIR + "/child_2.txt") + long_path = self.make_long_dir_path("test_ls_long_path") + + self.storage.mkdir(long_path, False) + self.storage.mkdir(long_path+ "/child_1", False) + self.make_small_file(long_path + "/child_2.txt") - ls = self.storage.ls(self.LONG_PATH_DIR) + ls = self.storage.ls(long_path) self.assertEqual(2, len(ls)) @@ -418,22 +432,22 @@ def test_ls_long_path(self): child2 = next(filter(lambda x: x.file_name == "child_2.txt", ls), None) self.assertIsNotNone(child1) - self.assertEqual(self.LONG_PATH_DIR + "/child_1", child1.storage_path) + self.assertEqual(long_path + "/child_1", child1.storage_path) self.assertEqual(_storage.FileType.DIRECTORY, child1.file_type) self.assertIsNotNone(child2) - self.assertEqual(self.LONG_PATH_DIR + "/child_2.txt", child2.storage_path) + self.assertEqual(long_path + "/child_2.txt", child2.storage_path) self.assertEqual(_storage.FileType.FILE, child2.file_type) def test_ls_extensions(self): # Corner case - dir with an extension, file without extension - self.storage.mkdir("ls_extensions", False) - self.storage.mkdir("ls_extensions/child_1.dat", False) - self.make_small_file("ls_extensions/child_2_file") + self.storage.mkdir("test_ls_extensions", False) + self.storage.mkdir("test_ls_extensions/child_1.dat", False) + self.make_small_file("test_ls_extensions/child_2_file") - ls = self.storage.ls("ls_extensions") + ls = self.storage.ls("test_ls_extensions") self.assertEqual(2, len(ls)) @@ -441,22 +455,22 @@ def test_ls_extensions(self): child2 = next(filter(lambda x: x.file_name == "child_2_file", ls), None) self.assertIsNotNone(child1) - self.assertEqual("ls_extensions/child_1.dat", child1.storage_path) + self.assertEqual("test_ls_extensions/child_1.dat", child1.storage_path) self.assertEqual(_storage.FileType.DIRECTORY, child1.file_type) self.assertIsNotNone(child2) - self.assertEqual("ls_extensions/child_2_file", child2.storage_path) + self.assertEqual("test_ls_extensions/child_2_file", child2.storage_path) self.assertEqual(_storage.FileType.FILE, child2.file_type) def test_ls_trailing_slash(self): # Storage path should be accepted with or without trailing slash - self.storage.mkdir("ls_trailing_slash", False) - self.make_small_file("ls_trailing_slash/some_file.txt") + self.storage.mkdir("test_ls_trailing_slash", False) + self.make_small_file("test_ls_trailing_slash/some_file.txt") - ls1 = self.storage.ls("ls_trailing_slash") - ls2 = self.storage.ls("ls_trailing_slash/") + ls1 = self.storage.ls("test_ls_trailing_slash") + ls2 = self.storage.ls("test_ls_trailing_slash/") self.assertEqual(1, len(ls1)) self.assertEqual(1, len(ls2)) @@ -465,39 +479,39 @@ def test_ls_storage_root_allowed(self): # Ls is one operation that is allowed on the storage root! - self.storage.mkdir("test_dir", False) - self.make_small_file("test_file.txt") + self.storage.mkdir("test_ls_storage_root_allowed_dir", False) + self.make_small_file("test_ls_storage_root_allowed_file.txt") ls = self.storage.ls(".") self.assertTrue(len(ls) >= 2) - child1 = next(filter(lambda x: x.file_name == "test_dir", ls), None) - child2 = next(filter(lambda x: x.file_name == "test_file.txt", ls), None) + child1 = next(filter(lambda x: x.file_name == "test_ls_storage_root_allowed_dir", ls), None) + child2 = next(filter(lambda x: x.file_name == "test_ls_storage_root_allowed_file.txt", ls), None) self.assertIsNotNone(child1) - self.assertEqual("test_dir", child1.storage_path) + self.assertEqual("test_ls_storage_root_allowed_dir", child1.storage_path) self.assertEqual(_storage.FileType.DIRECTORY, child1.file_type) self.assertIsNotNone(child2) - self.assertEqual("test_file.txt", child2.storage_path) + self.assertEqual("test_ls_storage_root_allowed_file.txt", child2.storage_path) self.assertEqual(_storage.FileType.FILE, child2.file_type) def test_ls_file(self): # Calling LS on a file returns a list of just that one file - self.make_small_file("test_file") + self.make_small_file("test_ls_file") - ls = self.storage.ls("test_file") + ls = self.storage.ls("test_ls_file") self.assertEqual(1, len(ls)) stat = ls[0] self.assertEqual(_storage.FileType.FILE, stat.file_type) - self.assertEqual("test_file", stat.file_name) - self.assertEqual("test_file", stat.storage_path) + self.assertEqual("test_ls_file", stat.file_name) + self.assertEqual("test_ls_file", stat.storage_path) def test_ls_missing(self): @@ -517,30 +531,32 @@ def test_mkdir_ok(self): # Simplest case - create a single directory - self.storage.mkdir("test_dir", False) + self.storage.mkdir("test_mkdir_ok", False) # Creating a single child dir when the parent already exists - self.storage.mkdir("test_dir/child", False) + self.storage.mkdir("test_mkdir_ok/child", False) - dir_exists = self.storage.exists("test_dir") - child_exists = self.storage.exists("test_dir/child") + dir_exists = self.storage.exists("test_mkdir_ok") + child_exists = self.storage.exists("test_mkdir_ok/child") self.assertTrue(dir_exists) self.assertTrue(child_exists) def test_mkdir_long_path(self): + long_path = self.make_long_dir_path("test_mkdir_long_path") + # Simplest case - create a single directory - self.storage.mkdir(self.LONG_PATH_DIR, False) + self.storage.mkdir(long_path, False) # Creating a single child dir when the parent already exists - self.storage.mkdir(self.LONG_PATH_DIR + "/child", False) + self.storage.mkdir(long_path + "/child", False) - dir_exists = self.storage.exists(self.LONG_PATH_DIR) - child_exists = self.storage.exists(self.LONG_PATH_DIR + "/child") + dir_exists = self.storage.exists(long_path) + child_exists = self.storage.exists(long_path + "/child") self.assertTrue(dir_exists) self.assertTrue(child_exists) @@ -549,33 +565,33 @@ def test_mkdir_dir_exists(self): # It is not an error to call mkdir on an existing directory - self.storage.mkdir("test_dir", False) + self.storage.mkdir("test_mkdir_dir_exists", False) - dir_exists_1 = self.storage.exists("test_dir") + dir_exists_1 = self.storage.exists("test_mkdir_dir_exists") self.assertTrue(dir_exists_1) - self.storage.mkdir("test_dir", False) + self.storage.mkdir("test_mkdir_dir_exists", False) - dir_exists_2 = self.storage.exists("test_dir") + dir_exists_2 = self.storage.exists("test_mkdir_dir_exists") self.assertTrue(dir_exists_2) def test_mkdir_file_exists(self): # mkdir should always fail if requested dir already exists and is a file - self.make_small_file("test_dir") + self.make_small_file("test_mkdir_file_exists") - self.assertRaises(_ex.EStorageRequest, lambda: self.storage.mkdir("test_dir", False)) + self.assertRaises(_ex.EStorageRequest, lambda: self.storage.mkdir("test_mkdir_file_exists", False)) def test_mkdir_missing_parent(self): # With recursive = false, mkdir with a missing parent should fail # Neither parent nor child dir should be created - self.assertRaises(_ex.EStorageRequest, lambda: self.storage.mkdir("test_dir/child", False)) + self.assertRaises(_ex.EStorageRequest, lambda: self.storage.mkdir("test_mkdir_missing_parent/child", False)) - dir_exists = self.storage.exists("test_dir") - child_exists = self.storage.exists("test_dir/child") + dir_exists = self.storage.exists("test_mkdir_missing_parent") + child_exists = self.storage.exists("test_mkdir_missing_parent/child") self.assertFalse(dir_exists) self.assertFalse(child_exists) @@ -584,10 +600,10 @@ def test_mkdir_recursive_ok(self): # mkdir, recursive = true, create parent and child dir in a single call - self.storage.mkdir("test_dir/child", True) + self.storage.mkdir("test_mkdir_recursive_ok/child", True) - dir_exists = self.storage.exists("test_dir") - child_exists = self.storage.exists("test_dir/child") + dir_exists = self.storage.exists("test_mkdir_recursive_ok") + child_exists = self.storage.exists("test_mkdir_recursive_ok/child") self.assertTrue(dir_exists) self.assertTrue(child_exists) @@ -596,12 +612,12 @@ def test_mkdir_recursive_dir_exists(self): # mkdir, when recursive = true it is not an error if the target dir already exists - self.storage.mkdir("test_dir/child", True) + self.storage.mkdir("test_mkdir_recursive_dir_exists/child", True) - self.storage.mkdir("test_dir/child", True) + self.storage.mkdir("test_mkdir_recursive_dir_exists/child", True) - dir_exists = self.storage.exists("test_dir") - child_exists = self.storage.exists("test_dir/child") + dir_exists = self.storage.exists("test_mkdir_recursive_dir_exists") + child_exists = self.storage.exists("test_mkdir_recursive_dir_exists/child") self.assertTrue(dir_exists) self.assertTrue(child_exists) @@ -610,10 +626,10 @@ def test_mkdir_recursive_file_exists(self): # mkdir should always fail if requested dir already exists and is a file - self.storage.mkdir("test_dir", False) - self.make_small_file("test_dir/child") + self.storage.mkdir("test_mkdir_recursive_file_exists", False) + self.make_small_file("test_mkdir_recursive_file_exists/child") - self.assertRaises(_ex.EStorageRequest, lambda: self.storage.mkdir("test_dir/child", True)) + self.assertRaises(_ex.EStorageRequest, lambda: self.storage.mkdir("test_mkdir_recursive_file_exists/child", True)) def test_mkdir_bad_paths(self): @@ -627,10 +643,10 @@ def test_mkdir_storage_root(self): def test_mkdir_unicode(self): - self.storage.mkdir("你好/你好", True) + self.storage.mkdir("test_mkdir_unicode/你好/你好", True) - dir_exists = self.storage.exists("你好") - child_exists = self.storage.exists("你好/你好") + dir_exists = self.storage.exists("test_mkdir_unicode/你好") + child_exists = self.storage.exists("test_mkdir_unicode/你好/你好") self.assertTrue(dir_exists) self.assertTrue(child_exists) @@ -643,59 +659,61 @@ def test_rm_ok(self): # Simplest case - create one file and delete it - self.make_small_file("test_file.txt") + self.make_small_file("test_rm_ok.txt") - self.storage.rm("test_file.txt") + self.storage.rm("test_rm_ok.txt") # File should be gone - exists = self.storage.exists("test_file.txt") + exists = self.storage.exists("test_rm_ok.txt") self.assertFalse(exists) def test_rm_long_path(self): + long_path = self.make_long_path("test_rm_long_path", ".txt") + # Simplest case - create one file and delete it - self.make_small_file(self.LONG_PATH_TXT_FILE) + self.make_small_file(long_path) - self.storage.rm(self.LONG_PATH_TXT_FILE) + self.storage.rm(long_path) # File should be gone - exists = self.storage.exists(self.LONG_PATH_TXT_FILE) + exists = self.storage.exists(long_path) self.assertFalse(exists) def test_rm_in_subdir(self): # Simplest case - create one file and delete it - self.make_small_file("sub_dir/test_file.txt") + self.make_small_file("test_rm_in_subdir/test_file.txt") - self.storage.rm("sub_dir/test_file.txt") + self.storage.rm("test_rm_in_subdir/test_file.txt") # File should be gone - exists = self.storage.exists("sub_dir/test_file.txt") + exists = self.storage.exists("test_rm_in_subdir/test_file.txt") self.assertFalse(exists) def test_rm_on_dir(self): # Calling rm on a directory is a bad request, even if the dir is empty - self.storage.mkdir("test_dir") + self.storage.mkdir("test_rm_on_dir") - self.assertRaises(_ex.EStorageRequest, lambda: self.storage.rm("test_dir")) + self.assertRaises(_ex.EStorageRequest, lambda: self.storage.rm("test_rm_on_dir")) # Dir should still exist because rm has failed - exists = self.storage.exists("test_dir") + exists = self.storage.exists("test_rm_on_dir") self.assertTrue(exists) def test_rm_missing(self): # Try to delete a path that does not exist - self.assertRaises(_ex.EStorageRequest, lambda: self.storage.rm("missing_path.dat")) + self.assertRaises(_ex.EStorageRequest, lambda: self.storage.rm("test_rm_missing.dat")) def test_rm_bad_paths(self): @@ -711,38 +729,40 @@ def test_rm_storage_root(self): def test_rmdir_ok(self): - self.storage.mkdir("test_dir", False) + self.storage.mkdir("test_rmdir_ok", False) - exists = self.storage.exists("test_dir") + exists = self.storage.exists("test_rmdir_ok") self.assertTrue(exists) - self.storage.rmdir("test_dir") + self.storage.rmdir("test_rmdir_ok") - exists = self.storage.exists("test_dir") + exists = self.storage.exists("test_rmdir_ok") self.assertFalse(exists) def test_rmdir_long_path(self): - self.storage.mkdir(self.LONG_PATH_DIR, False) + long_path = self.make_long_dir_path("test_rmdir_long_path") + + self.storage.mkdir(long_path, False) - exists = self.storage.exists(self.LONG_PATH_DIR,) + exists = self.storage.exists(long_path,) self.assertTrue(exists) - self.storage.rmdir(self.LONG_PATH_DIR,) + self.storage.rmdir(long_path,) - exists = self.storage.exists(self.LONG_PATH_DIR) + exists = self.storage.exists(long_path) self.assertFalse(exists) def test_rmdir_by_prefix(self): - self.storage.mkdir("test_dir/sub_dir", True) + self.storage.mkdir("test_rmdir_by_prefix/sub_dir", True) - exists = self.storage.exists("test_dir") + exists = self.storage.exists("test_rmdir_by_prefix") self.assertTrue(exists) - self.storage.rmdir("test_dir") + self.storage.rmdir("test_rmdir_by_prefix") - exists = self.storage.exists("test_dir") + exists = self.storage.exists("test_rmdir_by_prefix") self.assertFalse(exists) def test_rmdir_with_content(self): @@ -750,18 +770,18 @@ def test_rmdir_with_content(self): # Delete one whole dir tree # Sibling dir tree should be unaffected - self.storage.mkdir("test_dir/child_1", True) - self.storage.mkdir("test_dir/child_1/sub") - self.make_small_file("test_dir/child_1/file_a.txt") - self.make_small_file("test_dir/child_1/file_b.txt") - self.storage.mkdir("test_dir/child_2", True) - self.make_small_file("test_dir/child_2/file_a.txt") + self.storage.mkdir("test_rmdir_with_content/child_1", True) + self.storage.mkdir("test_rmdir_with_content/child_1/sub") + self.make_small_file("test_rmdir_with_content/child_1/file_a.txt") + self.make_small_file("test_rmdir_with_content/child_1/file_b.txt") + self.storage.mkdir("test_rmdir_with_content/child_2", True) + self.make_small_file("test_rmdir_with_content/child_2/file_a.txt") - self.storage.rmdir("test_dir/child_1") + self.storage.rmdir("test_rmdir_with_content/child_1") - exists1 = self.storage.exists("test_dir/child_1") - exists2 = self.storage.exists("test_dir/child_2") - size2a = self.storage.size("test_dir/child_2/file_a.txt") + exists1 = self.storage.exists("test_rmdir_with_content/child_1") + exists2 = self.storage.exists("test_rmdir_with_content/child_2") + size2a = self.storage.size("test_rmdir_with_content/child_2/file_a.txt") self.assertFalse(exists1) self.assertTrue(exists2) @@ -771,20 +791,20 @@ def test_rmdir_on_file(self): # Calling rmdir on a file is a bad request - self.make_small_file("test_file.txt") + self.make_small_file("test_rmdir_on_file.txt") - self.assertRaises(_ex.EStorageRequest, lambda: self.storage.rmdir("test_file.txt")) + self.assertRaises(_ex.EStorageRequest, lambda: self.storage.rmdir("test_rmdir_on_file.txt")) # File should still exist because rm has failed - exists = self.storage.exists("test_file.txt") + exists = self.storage.exists("test_rmdir_on_file.txt") self.assertTrue(exists) def test_rmdir_missing(self): # Try to delete a path that does not exist - self.assertRaises(_ex.EStorageRequest, lambda: self.storage.rmdir("missing_path")) + self.assertRaises(_ex.EStorageRequest, lambda: self.storage.rmdir("test_rmdir_missing")) def test_rmdir_bad_paths(self): @@ -854,7 +874,10 @@ class FileReadWriteTestSuite: fail = unittest.TestCase.fail # Windows limits individual path segments to 255 chars - LONG_PATH_TXT_FILE = "long_" + "A" * 246 + ".txt" + + @classmethod + def make_long_path(cls, prefix, suffix): + return prefix + "A" * (255 - len(prefix) - len(suffix)) + suffix def __init__(self): self.storage: _storage.IFileStorage = None # noqa @@ -865,7 +888,7 @@ def __init__(self): def test_round_trip_basic(self): - storage_path = "haiku.txt" + storage_path = "test_round_trip_basic.txt" haiku = \ "The data goes in;\n" + \ @@ -878,7 +901,7 @@ def test_round_trip_basic(self): def test_round_trip_long_path(self): - storage_path = self.LONG_PATH_TXT_FILE + storage_path = self.make_long_path("test_round_trip_long_path", ".txt") haiku = \ "The data goes in;\n" + \ @@ -891,7 +914,7 @@ def test_round_trip_long_path(self): def test_round_trip_large(self): - storage_path = "test_file.dat" + storage_path = "test_round_trip_large.dat" # One 10 M chunk bytes_ = random.randbytes(10 * 1024 * 1024) @@ -900,7 +923,7 @@ def test_round_trip_large(self): def test_round_trip_heterogeneous(self): - storage_path = "test_file.dat" + storage_path = "test_round_trip_heterogeneous.dat" # Selection of different size chunks bytes_ = [ @@ -916,7 +939,7 @@ def test_round_trip_heterogeneous(self): def test_round_trip_empty(self): - storage_path = "test_file.dat" + storage_path = "test_round_trip_empty.dat" empty_bytes = bytes() self.do_round_trip(storage_path, [empty_bytes], self.storage) @@ -929,7 +952,7 @@ def test_round_trip_unicode(self): "白毛浮绿水，\n" + \ "红掌拨清波" - storage_path = "咏鹅.txt" + storage_path = "test_round_trip_unicode/咏鹅.txt" storage_bytes = an_ode_to_the_goose.encode('utf-8') self.do_round_trip(storage_path, [storage_bytes], self.storage) @@ -959,18 +982,18 @@ def test_write_missing_dir(self): # Writing a file will always create the parent dir if it doesn't already exist # This is in line with cloud bucket semantics - storage_path = "missing_dir/some_file.txt" + storage_path = "test_write_missing_dir/some_file.txt" content = "Some content".encode('utf-8') self.storage.write_bytes(storage_path, content) - dir_exists = self.storage.exists("missing_dir") + dir_exists = self.storage.exists("test_write_missing_dir") file_exists = self.storage.exists(storage_path) self.assertTrue(dir_exists) self.assertTrue(file_exists) - dir_stat = self.storage.stat("missing_dir") + dir_stat = self.storage.stat("test_write_missing_dir") file_stat = self.storage.stat(storage_path) self.assertEqual(_storage.FileType.DIRECTORY, dir_stat.file_type) @@ -981,7 +1004,7 @@ def test_write_file_already_exists(self): # Writing a file always overwrites any existing content # This is in line with cloud bucket semantics - storage_path = "some_file.txt" + storage_path = "test_write_file_already_exists.txt" content = "Some content that is longer than what will be written later".encode('utf-8') def write_to_path(path_, content_): @@ -1005,7 +1028,7 @@ def test_write_dir_already_exists(self): # File storage should not allow a file to be written if a dir exists with the same name # TRAC prohibits this even though it is allowed in pure bucket semantics - storage_path = "some_file.txt" + storage_path = "test_write_dir_already_exists.txt" self.storage.mkdir(storage_path) @@ -1044,13 +1067,13 @@ def test_write_outside_root(self): def test_read_missing(self): - storage_path = "missing_file.txt" + storage_path = "test_read_missing.txt" self.assertRaises(_ex.EStorageRequest, lambda: self.storage.read_byte_stream(storage_path)) def test_read_dir(self): - storage_path = "not_a_file/" + storage_path = "test_read_dir/" self.storage.mkdir(storage_path) @@ -1102,7 +1125,7 @@ class TestException(Exception): def test_write_error_immediately(self): - storage_path = "test_file.dat" + storage_path = "test_write_error_immediately.dat" def attempt_write(): with self.storage.write_byte_stream(storage_path): @@ -1118,7 +1141,7 @@ def attempt_write(): def test_write_error_after_chunk(self): - storage_path = "test_file.dat" + storage_path = "test_write_error_after_chunk.dat" first_chunk = "Some content\n".encode('utf-8') def attempt_write(): @@ -1136,7 +1159,7 @@ def attempt_write(): def test_write_error_then_retry(self): - storage_path = "test_file.dat" + storage_path = "test_write_error_then_retry.dat" chunk_size = 10000 chunk = random.randbytes(chunk_size) @@ -1167,7 +1190,7 @@ def test_read_concurrent_streams(self): # Allow two read streams to be opened concurrently to the same file - storage_path = "some_file.txt" + storage_path = "test_read_concurrent_streams.txt" content = "Some content".encode('utf-8') self.storage.write_bytes(storage_path, content) @@ -1182,7 +1205,7 @@ def test_read_concurrent_streams(self): def test_read_delete_while_open(self): - storage_path = "some_file.txt" + storage_path = "test_read_delete_while_open.txt" content = random.randbytes(64 * 1024) self.storage.write_bytes(storage_path, content) @@ -1218,7 +1241,7 @@ def test_read_delete_while_open(self): def test_read_cancel_immediately(self): - storage_path = "some_file.txt" + storage_path = "test_read_cancel_immediately.txt" content = "Some content".encode('utf-8') self.storage.write_bytes(storage_path, content) @@ -1235,7 +1258,7 @@ def test_read_cancel_and_retry(self): # Create a file big enough that it can't be read in a single chunk - storage_path = "some_file.txt" + storage_path = "test_read_cancel_and_retry.txt" content = random.randbytes(10000) self.storage.write_bytes(storage_path, content) @@ -1263,7 +1286,7 @@ def test_read_cancel_and_retry(self): def test_read_cancel_and_delete(self): - storage_path = "some_file.txt" + storage_path = "test_read_cancel_and_delete.txt" content = "Some content".encode('utf-8') self.storage.write_bytes(storage_path, content)