Merge remote-tracking branch 'origin/master' into global-config-doc

.github/ISSUE_TEMPLATE/bug_report.md → .github/ISSUE_TEMPLATE/issue_report.md

@@ -1,5 +1,5 @@
 ---
-name: Bug report
+name: Issue report
 about: Create a report to help us improve
 title: ''
 labels: ''

.github/workflows/unit-tests.yml

@@ -18,10 +18,12 @@ jobs:
       TERM: dumb
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
       - uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
-      - uses: actions/cache@v2
+      - uses: actions/cache@v2.1.4
         with:
           path: |
             ~/.m2/repository
@@ -32,4 +34,8 @@ jobs:
             ${{ runner.os }}-gradle-
       - name: Run tests
         run: |
-          ./gradlew clean build --stacktrace
+          ./gradlew clean build jacocoTestReport --stacktrace
+      - name: Test Coverage
+        uses: codecov/codecov-action@v1
+        with:
+          name: actions ${{matrix.java}}

build.gradle

@@ -8,7 +8,7 @@ plugins {
   id 'io.freefair.maven-plugin' version '5.3.0' apply false
 
   // apply so we can correctly configure the test runner to be gradle at the project level
-  id 'org.jetbrains.gradle.plugin.idea-ext' version '0.10'
+  id 'org.jetbrains.gradle.plugin.idea-ext' version '1.0'
 }
 
 // run tests in intellij using gradle test runner
@@ -41,6 +41,7 @@ subprojects {
   apply plugin: 'com.github.sherter.google-java-format'
   apply plugin: 'net.ltgt.apt'
   apply plugin: 'net.ltgt.errorprone'
+  apply plugin: 'jacoco'
 
   sourceCompatibility = JavaVersion.VERSION_1_8
   targetCompatibility = JavaVersion.VERSION_1_8
@@ -71,7 +72,7 @@ subprojects {
     JACKSON_DATABIND: 'com.fasterxml.jackson.core:jackson-databind:2.12.1',
     JACKSON_DATAFORMAT_YAML: 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.12.1',
     JACKSON_DATATYPE_JSR310: 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.12.1',
-    ASM: 'org.ow2.asm:asm:9.0',
+    ASM: 'org.ow2.asm:asm:9.1',
     PICOCLI: 'info.picocli:picocli:4.6.1',
 
     MAVEN_API: 'org.apache.maven:maven-plugin-api:3.6.3',
@@ -417,6 +418,15 @@ subprojects {
       }
     }
   }
+
+  /* Add Test Coverage */
+  jacocoTestReport {
+    reports {
+      xml.enabled true
+      html.enabled false
+    }
+  }
+
   /* INCLUDED PROJECT DEPENDENCY HELPER */
 
   /* LOCAL DEVELOPMENT HELPER TASKS */

examples/README.md

@@ -39,5 +39,5 @@ See [java-agent](java-agent) for launching with the Stackdriver Debugger Java ag
 ### Kafka Connect+Streams example
 
 See the following projects for using Jib-packaged applications in coordination with [Apache Kafka](http://kafka.apache.org/documentation).
-  - [Kakfa Streams](http://kafka.apache.org/documentation/streams) - [`OneCricketeer/kafka-streams-jib-example`](https://github.com/OneCricketeer/kafka-streams-jib-example)
+  - [Kafka Streams](http://kafka.apache.org/documentation/streams) - [`OneCricketeer/kafka-streams-jib-example`](https://github.com/OneCricketeer/kafka-streams-jib-example)
   - [Kafka Connect](http://kafka.apache.org/documentation#connect) - [`OneCricketeer/apache-kafka-connect-docker`](https://github.com/OneCricketeer/apache-kafka-connect-docker)

jib-cli/src/main/java/com/google/cloud/tools/jib/cli/jar/SpringBootExplodedProcessor.java

@@ -55,7 +55,7 @@ public List<FileEntriesLayer> createLayers() throws IOException {
     }
     try (JarFile jarFile = new JarFile(jarPath.toFile())) {
       Path localExplodedJarRoot = tempDirectoryPath;
-      ZipUtil.unzip(jarPath, localExplodedJarRoot);
+      ZipUtil.unzip(jarPath, localExplodedJarRoot, true);
       ZipEntry layerIndex = jarFile.getEntry("BOOT-INF/layers.idx");
       if (layerIndex != null) {
         return createLayersForLayeredSpringBootJar(localExplodedJarRoot);

jib-cli/src/main/java/com/google/cloud/tools/jib/cli/jar/StandardExplodedProcessor.java

@@ -52,7 +52,7 @@ public List<FileEntriesLayer> createLayers() throws IOException {
     // Determine class and resource files in the directory containing jar contents and create
     // FileEntriesLayer for each type of layer (classes or resources).
     Path localExplodedJarRoot = tempDirectoryPath;
-    ZipUtil.unzip(jarPath, localExplodedJarRoot);
+    ZipUtil.unzip(jarPath, localExplodedJarRoot, true);
     Predicate<Path> isClassFile = path -> path.getFileName().toString().endsWith(".class");
     Predicate<Path> isResourceFile = isClassFile.negate().and(Files::isRegularFile);
     FileEntriesLayer classesLayer =

jib-core/build.gradle

@@ -17,6 +17,7 @@ dependencies {
   implementation dependencyStrings.ASM
 
   testImplementation dependencyStrings.JUNIT
+  testImplementation dependencyStrings.TRUTH
   testImplementation dependencyStrings.MOCKITO_CORE
   testImplementation dependencyStrings.SLF4J_API
   testImplementation dependencyStrings.SYSTEM_RULES

jib-core/src/main/java/com/google/cloud/tools/jib/tar/TarExtractor.java

@@ -16,6 +16,7 @@
 
 package com.google.cloud.tools.jib.tar;
 
+import com.google.cloud.tools.jib.filesystem.DirectoryWalker;
 import com.google.common.io.ByteStreams;
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
@@ -26,6 +27,9 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.attribute.FileTime;
+import java.util.ArrayList;
+import java.util.List;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 
@@ -40,22 +44,42 @@ public class TarExtractor {
    * @throws IOException if extraction fails
    */
   public static void extract(Path source, Path destination) throws IOException {
-    String canonicalDestination = destination.toFile().getCanonicalPath();
+    extract(source, destination, false);
+  }
 
+  /**
+   * Extracts a tarball to the specified destination.
+   *
+   * @param source the tarball to extract
+   * @param destination the output directory
+   * @param enableReproducibleTimestamps whether or not reproducible timestamps should be used
+   * @throws IOException if extraction fails
+   * @throws IllegalStateException when reproducible timestamps are enabled but the target root used
+   *     for extracting the tar contents is not empty
+   */
+  public static void extract(Path source, Path destination, boolean enableReproducibleTimestamps)
+      throws IOException {
+    if (enableReproducibleTimestamps
+        && Files.isDirectory(destination)
+        && destination.toFile().list().length != 0) {
+      throw new IllegalStateException(
+          "Cannot enable reproducible timestamps. They can only be enabled when the target root doesn't exist or is an empty directory");
+    }
+    String canonicalDestination = destination.toFile().getCanonicalPath();
+    List<TarArchiveEntry> entries = new ArrayList<>();
     try (InputStream in = new BufferedInputStream(Files.newInputStream(source));
         TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(in)) {
-
       for (TarArchiveEntry entry = tarArchiveInputStream.getNextTarEntry();
           entry != null;
           entry = tarArchiveInputStream.getNextTarEntry()) {
+        entries.add(entry);
         Path entryPath = destination.resolve(entry.getName());
 
         String canonicalTarget = entryPath.toFile().getCanonicalPath();
         if (!canonicalTarget.startsWith(canonicalDestination + File.separator)) {
           String offender = entry.getName() + " from " + source;
           throw new IOException("Blocked unzipping files outside destination: " + offender);
         }
-
         if (entry.isDirectory()) {
           Files.createDirectories(entryPath);
         } else {
@@ -73,5 +97,37 @@ public static void extract(Path source, Path destination) throws IOException {
         }
       }
     }
+    preserveModificationTimes(destination, entries, enableReproducibleTimestamps);
+  }
+
+  /**
+   * Preserve modification timestamps of files and directories in a tar file. If a directory is not
+   * an entry in the tar file and reproducible timestamps are enabled then its modification
+   * timestamp is set to a constant value. Note that the modification timestamps of symbolic links
+   * are not preserved even with reproducible timestamps enabled.
+   *
+   * @param destination target root for unzipping
+   * @param entries list of entries in tar file
+   * @param enableReproducibleTimestamps whether or not reproducible timestamps should be used
+   * @throws IOException when I/O error occurs
+   */
+  private static void preserveModificationTimes(
+      Path destination, List<TarArchiveEntry> entries, boolean enableReproducibleTimestamps)
+      throws IOException {
+    if (enableReproducibleTimestamps) {
+      FileTime epochPlusOne = FileTime.fromMillis(1000L);
+      new DirectoryWalker(destination)
+          .filter(Files::isDirectory)
+          .walk(path -> Files.setLastModifiedTime(path, epochPlusOne));
+    }
+    for (TarArchiveEntry entry : entries) {
+
+      // Setting the symbolic link's modification timestamp will cause the modification timestamp of
+      // the target to change
+      if (!entry.isSymbolicLink()) {
+        Files.setLastModifiedTime(
+            destination.resolve(entry.getName()), FileTime.from(entry.getModTime().toInstant()));
+      }
+    }
   }
 }

jib-core/src/test/java/com/google/cloud/tools/jib/tar/TarExtractorTest.java

@@ -16,12 +16,17 @@
 
 package com.google.cloud.tools.jib.tar;
 
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
 import com.google.common.io.Resources;
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.attribute.FileTime;
+import java.time.Instant;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.junit.Assert;
@@ -79,4 +84,54 @@ public void testExtract_symlinks() throws URISyntaxException, IOException {
     Assert.assertTrue(Files.isSymbolicLink(destination.resolve("directory-symlink")));
     Assert.assertTrue(Files.isSymbolicLink(destination.resolve("directory1/file-symlink")));
   }
+
+  @Test
+  public void testExtract_modificationTimePreserved() throws URISyntaxException, IOException {
+    Path source = Paths.get(Resources.getResource("core/extract.tar").toURI());
+    Path destination = temporaryFolder.getRoot().toPath();
+
+    TarExtractor.extract(source, destination);
+
+    assertThat(Files.getLastModifiedTime(destination.resolve("file A")))
+        .isEqualTo(FileTime.from(Instant.parse("2019-08-01T16:13:09Z")));
+    assertThat(Files.getLastModifiedTime(destination.resolve("file B")))
+        .isEqualTo(FileTime.from(Instant.parse("2019-08-01T16:12:00Z")));
+    assertThat(Files.getLastModifiedTime(destination.resolve("folder")))
+        .isEqualTo(FileTime.from(Instant.parse("2019-08-01T16:12:33Z")));
+    assertThat(Files.getLastModifiedTime(destination.resolve("folder/nested folder")))
+        .isEqualTo(FileTime.from(Instant.parse("2019-08-01T16:13:30Z")));
+    assertThat(Files.getLastModifiedTime(destination.resolve("folder/nested folder/file C")))
+        .isEqualTo(FileTime.from(Instant.parse("2019-08-01T16:12:21Z")));
+  }
+
+  @Test
+  public void testExtract_reproducibleTimestampsEnabled() throws URISyntaxException, IOException {
+    // The tarfile has only level1/level2/level3/file.txt packaged
+    Path source = Paths.get(Resources.getResource("core/tarfile-only-file-packaged.tar").toURI());
+
+    Path destination = temporaryFolder.getRoot().toPath();
+
+    TarExtractor.extract(source, destination, true);
+
+    assertThat(Files.getLastModifiedTime(destination.resolve("level-1")))
+        .isEqualTo(FileTime.fromMillis(1000L));
+    assertThat(Files.getLastModifiedTime(destination.resolve("level-1/level-2")))
+        .isEqualTo(FileTime.fromMillis(1000L));
+    assertThat(Files.getLastModifiedTime(destination.resolve("level-1/level-2/level-3")))
+        .isEqualTo(FileTime.fromMillis(1000L));
+    assertThat(Files.getLastModifiedTime(destination.resolve("level-1/level-2/level-3/file.txt")))
+        .isEqualTo(FileTime.from(Instant.parse("2021-01-29T21:10:02Z")));
+  }
+
+  @Test
+  public void testExtract_reproducibleTimestampsEnabled_destinationNotEmpty() throws IOException {
+    Path destination = temporaryFolder.getRoot().toPath();
+    temporaryFolder.newFile();
+
+    IllegalStateException exception =
+        assertThrows(
+            IllegalStateException.class,
+            () -> TarExtractor.extract(Paths.get("ignore"), destination, true));
+    assertThat(exception).hasMessageThat().startsWith("Cannot enable reproducible timestamps");
+  }
 }

jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/ZipUtil.java

@@ -16,6 +16,7 @@
 
 package com.google.cloud.tools.jib.plugins.common;
 
+import com.google.cloud.tools.jib.filesystem.DirectoryWalker;
 import com.google.common.io.ByteStreams;
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
@@ -25,6 +26,9 @@
 import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.attribute.FileTime;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
@@ -39,12 +43,33 @@ public class ZipUtil {
    * @throws IOException when I/O error occurs
    */
   public static void unzip(Path archive, Path destination) throws IOException {
-    String canonicalDestination = destination.toFile().getCanonicalPath();
+    unzip(archive, destination, false);
+  }
 
+  /**
+   * Unzips {@code archive} into {@code destination}.
+   *
+   * @param archive zip archive to unzip
+   * @param destination target root for unzipping
+   * @param enableReproducibleTimestamps whether or not reproducible timestamps should be used
+   * @throws IOException when I/O error occurs
+   * @throws IllegalStateException when reproducible timestamps are enabled but the target root used
+   *     for unzipping is not empty
+   */
+  public static void unzip(Path archive, Path destination, boolean enableReproducibleTimestamps)
+      throws IOException {
+    if (enableReproducibleTimestamps
+        && Files.isDirectory(destination)
+        && destination.toFile().list().length != 0) {
+      throw new IllegalStateException(
+          "Cannot enable reproducible timestamps. They can only be enabled when the target root doesn't exist or is an empty directory");
+    }
+    String canonicalDestination = destination.toFile().getCanonicalPath();
+    List<ZipEntry> entries = new ArrayList<>();
     try (InputStream fileIn = new BufferedInputStream(Files.newInputStream(archive));
         ZipInputStream zipIn = new ZipInputStream(fileIn)) {
-
       for (ZipEntry entry = zipIn.getNextEntry(); entry != null; entry = zipIn.getNextEntry()) {
+        entries.add(entry);
         Path entryPath = destination.resolve(entry.getName());
 
         String canonicalTarget = entryPath.toFile().getCanonicalPath();
@@ -65,5 +90,30 @@ public static void unzip(Path archive, Path destination) throws IOException {
         }
       }
     }
+    preserveModificationTimes(destination, entries, enableReproducibleTimestamps);
+  }
+
+  /**
+   * Preserve modification time of files and directories in a zip file. If a directory is not an
+   * entry in the zip file and reproducible timestamps are enabled then its modification timestamp
+   * is set to a constant value.
+   *
+   * @param destination target root for unzipping
+   * @param entries list of entries in zip file
+   * @param enableReproducibleTimestamps whether or not reproducible timestamps should be used
+   * @throws IOException when I/O error occurs
+   */
+  private static void preserveModificationTimes(
+      Path destination, List<ZipEntry> entries, boolean enableReproducibleTimestamps)
+      throws IOException {
+    if (enableReproducibleTimestamps) {
+      FileTime epochPlusOne = FileTime.fromMillis(1000L);
+      new DirectoryWalker(destination)
+          .filter(Files::isDirectory)
+          .walk(path -> Files.setLastModifiedTime(path, epochPlusOne));
+    }
+    for (ZipEntry entry : entries) {
+      Files.setLastModifiedTime(destination.resolve(entry.getName()), entry.getLastModifiedTime());
+    }
   }
 }