From b2bd6b4d6ffe5590798cfa93a4e2518cf6d37b76 Mon Sep 17 00:00:00 2001 From: Q Chen Date: Wed, 26 Sep 2018 16:31:52 -0700 Subject: [PATCH] Stores temporary cache files under cache directory. (#1037) --- .../jib/filesystem/TemporaryDirectory.java | 7 +++--- .../jib/ncache/DefaultCacheStorageFiles.java | 10 ++++++++ .../jib/ncache/DefaultCacheStorageWriter.java | 23 +++++++++++++++++-- .../filesystem/TemporaryDirectoryTest.java | 6 +++-- .../ncache/DefaultCacheStorageFilesTest.java | 6 +++++ 5 files changed, 45 insertions(+), 7 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/TemporaryDirectory.java b/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/TemporaryDirectory.java index d4db8d84a2..b55d573fdb 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/TemporaryDirectory.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/TemporaryDirectory.java @@ -32,12 +32,13 @@ public class TemporaryDirectory implements Closeable { private final Path temporaryDirectory; /** - * Creates a new temporary directory. + * Creates a new temporary directory under an existing {@code parentDirectory}. * + * @param parentDirectory the directory to create the temporary directory within * @throws IOException if an I/O exception occurs */ - public TemporaryDirectory() throws IOException { - temporaryDirectory = Files.createTempDirectory(null); + public TemporaryDirectory(Path parentDirectory) throws IOException { + temporaryDirectory = Files.createTempDirectory(parentDirectory, null); } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageFiles.java b/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageFiles.java index ae901d47d9..670718eb1e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageFiles.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageFiles.java @@ -26,6 +26,7 @@ class DefaultCacheStorageFiles { private static final String LAYERS_DIRECTORY = "layers"; private static final String METADATA_FILENAME = "metadata"; private static final String SELECTORS_DIRECTORY = "selectors"; + private static final String TEMPORARY_DIRECTORY = "tmp"; /** * Returns whether or not {@code file} is a layer contents file. @@ -140,4 +141,13 @@ Path getLayersDirectory() { Path getLayerDirectory(DescriptorDigest layerDigest) { return getLayersDirectory().resolve(layerDigest.getHash()); } + + /** + * Gets the directory to store temporary files. + * + * @return the directory for temporary files + */ + Path getTemporaryDirectory() { + return cacheDirectory.resolve(TEMPORARY_DIRECTORY); + } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageWriter.java index 91de02cd09..2fca700768 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageWriter.java @@ -25,10 +25,13 @@ import java.io.BufferedOutputStream; import java.io.IOException; import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.DirectoryNotEmptyException; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.GZIPOutputStream; /** Writes to the default cache storage engine. */ @@ -59,6 +62,10 @@ private WrittenLayer( * @throws IOException if an I/O exception occurs */ private static void moveIfDoesNotExist(Path source, Path destination) throws IOException { + if (Files.exists(destination)) { + return; + } + try { Files.move(source, destination, StandardCopyOption.ATOMIC_MOVE); @@ -75,6 +82,15 @@ private static void moveIfDoesNotExist(Path source, Path destination) throws IOE } catch (FileAlreadyExistsException alsoIgnored) { // Same reasoning + + } catch (DirectoryNotEmptyException ex) { + // The file system cannot rename directories, so we must resort to copying the directory. + Files.createDirectory(destination); + try (Stream sourceFiles = Files.list(source)) { + for (Path sourceFile : sourceFiles.collect(Collectors.toList())) { + Files.copy(sourceFile, destination.resolve(sourceFile.getFileName())); + } + } } } } @@ -108,8 +124,11 @@ CacheEntry write(CacheWrite cacheWrite) throws IOException { // Creates the layers directory if it doesn't exist. Files.createDirectories(defaultCacheStorageFiles.getLayersDirectory()); - // Creates the temporary directory. - try (TemporaryDirectory temporaryDirectory = new TemporaryDirectory()) { + // Creates the temporary directory. The temporary directory must be in the same FileStore as the + // final location for Files.move to work. + Files.createDirectories(defaultCacheStorageFiles.getTemporaryDirectory()); + try (TemporaryDirectory temporaryDirectory = + new TemporaryDirectory(defaultCacheStorageFiles.getTemporaryDirectory())) { Path temporaryLayerDirectory = temporaryDirectory.getDirectory(); // Writes the layer file to the temporary directory. diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/filesystem/TemporaryDirectoryTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/filesystem/TemporaryDirectoryTest.java index 338aaa53c1..1d41843e04 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/filesystem/TemporaryDirectoryTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/filesystem/TemporaryDirectoryTest.java @@ -42,7 +42,8 @@ private static void createFilesInDirectory(Path directory) @Test public void testClose_directoryDeleted() throws IOException, URISyntaxException { - try (TemporaryDirectory temporaryDirectory = new TemporaryDirectory()) { + try (TemporaryDirectory temporaryDirectory = + new TemporaryDirectory(temporaryFolder.newFolder().toPath())) { createFilesInDirectory(temporaryDirectory.getDirectory()); temporaryDirectory.close(); @@ -54,7 +55,8 @@ public void testClose_directoryDeleted() throws IOException, URISyntaxException public void testClose_directoryNotDeletedIfMoved() throws IOException, URISyntaxException { Path destinationParent = temporaryFolder.newFolder().toPath(); - try (TemporaryDirectory temporaryDirectory = new TemporaryDirectory()) { + try (TemporaryDirectory temporaryDirectory = + new TemporaryDirectory(temporaryFolder.newFolder().toPath())) { createFilesInDirectory(temporaryDirectory.getDirectory()); Assert.assertFalse(Files.exists(destinationParent.resolve("destination"))); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageFilesTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageFilesTest.java index 9324c0c91b..250c50baaa 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageFilesTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/ncache/DefaultCacheStorageFilesTest.java @@ -178,4 +178,10 @@ public void testGetLayerDirectory() throws DigestException { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), testDefaultCacheStorageFiles.getLayerDirectory(layerDigest)); } + + @Test + public void testGetTemporaryDirectory() { + Assert.assertEquals( + Paths.get("cache/directory/tmp"), testDefaultCacheStorageFiles.getTemporaryDirectory()); + } }