Add ExtractTarStep and SaveDockerStep

jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java

@@ -23,6 +23,7 @@
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.util.zip.GZIPOutputStream;
 
 /** Static methods for {@link Blob}. */
 public class Blobs {
@@ -77,5 +78,20 @@ public static byte[] writeToByteArray(Blob blob) throws IOException {
     return byteArrayOutputStream.toByteArray();
   }
 
+  /**
+   * Gets a {@link Blob} that is {@code blob} compressed.
+   *
+   * @param blob the {@link Blob} to compress
+   * @return the compressed {@link Blob}
+   */
+  public static Blob compress(Blob blob) {
+    return Blobs.from(
+        outputStream -> {
+          try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream)) {
+            blob.writeTo(compressorStream);
+          }
+        });
+  }
+
   private Blobs() {}
 }

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java

@@ -0,0 +1,150 @@
+/*
+ * Copyright 2019 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.cloud.tools.jib.builder.steps;
+
+import com.fasterxml.jackson.databind.MapperFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.cloud.tools.jib.blob.Blob;
+import com.google.cloud.tools.jib.blob.BlobDescriptor;
+import com.google.cloud.tools.jib.blob.Blobs;
+import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage;
+import com.google.cloud.tools.jib.cache.CachedLayer;
+import com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate;
+import com.google.cloud.tools.jib.filesystem.FileOperations;
+import com.google.cloud.tools.jib.image.Image;
+import com.google.cloud.tools.jib.image.LayerCountMismatchException;
+import com.google.cloud.tools.jib.image.json.BadContainerConfigurationFormatException;
+import com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;
+import com.google.cloud.tools.jib.image.json.JsonToImageTranslator;
+import com.google.cloud.tools.jib.image.json.V22ManifestTemplate;
+import com.google.cloud.tools.jib.json.JsonTemplateMapper;
+import com.google.cloud.tools.jib.tar.TarExtractor;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.ByteStreams;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.zip.GZIPInputStream;
+
+/** Extracts a tar file base image. */
+public class ExtractTarStep implements Callable<LocalImage> {
+
+  /** Contains an {@link Image} and its layers. * */
+  static class LocalImage {
+    final Image baseImage;
+    final List<PreparedLayer> layers;
+
+    LocalImage(Image baseImage, List<PreparedLayer> layers) {
+      this.baseImage = baseImage;
+      this.layers = layers;
+    }
+  }
+
+  /**
+   * Checks the first two bytes of a file to see if it has been gzipped.
+   *
+   * @param path the file to check
+   * @return {@code true} if the file is gzipped, {@code false} if not
+   * @throws IOException if reading the file fails
+   * @see <a href="http://www.zlib.org/rfc-gzip.html#file-format">GZIP file format</a>
+   */
+  @VisibleForTesting
+  static boolean isGzipped(Path path) throws IOException {
+    try (InputStream inputStream = Files.newInputStream(path)) {
+      inputStream.mark(2);
+      int magic = (inputStream.read() & 0xff) | ((inputStream.read() << 8) & 0xff00);
+      return magic == GZIPInputStream.GZIP_MAGIC;
+    }
+  }
+
+  private final Path tarPath;
+  private final Path destination;
+
+  ExtractTarStep(Path tarPath, Path destination) {
+    this.tarPath = tarPath;
+    this.destination = destination;
+  }
+
+  @Override
+  public LocalImage call()
+      throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException {
+    Files.createDirectories(destination);
+    FileOperations.deleteRecursiveOnExit(destination);
+    TarExtractor.extract(tarPath, destination);
+
+    InputStream manifestStream = Files.newInputStream(destination.resolve("manifest.json"));
+    DockerManifestEntryTemplate loadManifest =
+        new ObjectMapper()
+            .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true)
+            .readValue(manifestStream, DockerManifestEntryTemplate[].class)[0];
+    manifestStream.close();
+    ContainerConfigurationTemplate configurationTemplate =
+        JsonTemplateMapper.readJsonFromFile(
+            destination.resolve(loadManifest.getConfig()), ContainerConfigurationTemplate.class);
+
+    List<String> layerFiles = loadManifest.getLayerFiles();
+    if (configurationTemplate.getLayerCount() != layerFiles.size()) {
+      throw new LayerCountMismatchException(
+          "Invalid base image format: manifest contains "
+              + layerFiles.size()
+              + " layers, but container configuration contains "
+              + configurationTemplate.getLayerCount()
+              + " layers");
+    }
+
+    // Check the first layer to see if the layers are compressed already. 'docker save' output is
+    // uncompressed, but a jib-built tar has compressed layers.
+    boolean layersAreCompressed =
+        layerFiles.size() > 0 && isGzipped(destination.resolve(layerFiles.get(0)));
+
+    // Process layer blobs
+    // TODO: Optimize; compressing/calculating layer digests is slow
+    List<PreparedLayer> layers = new ArrayList<>();
+    V22ManifestTemplate v22Manifest = new V22ManifestTemplate();
+    for (int index = 0; index < layerFiles.size(); index++) {
+      Path file = destination.resolve(layerFiles.get(index));
+
+      // Compress layers if necessary and calculate the digest/size
+      Blob blob = layersAreCompressed ? Blobs.from(file) : Blobs.compress(Blobs.from(file));
+      BlobDescriptor blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream());
+
+      // 'manifest' contains the layer files in the same order as the diff ids in 'configuration',
+      // so we don't need to recalculate those.
+      // https://containers.gitbook.io/build-containers-the-hard-way/#docker-load-format
+      CachedLayer layer =
+          CachedLayer.builder()
+              .setLayerBlob(blob)
+              .setLayerDigest(blobDescriptor.getDigest())
+              .setLayerSize(blobDescriptor.getSize())
+              .setLayerDiffId(configurationTemplate.getLayerDiffId(index))
+              .build();
+
+      layers.add(new PreparedLayer.Builder(layer).build());
+      v22Manifest.addLayer(blobDescriptor.getSize(), blobDescriptor.getDigest());
+    }
+
+    BlobDescriptor configDescriptor =
+        Blobs.from(configurationTemplate).writeTo(ByteStreams.nullOutputStream());
+    v22Manifest.setContainerConfiguration(configDescriptor.getSize(), configDescriptor.getDigest());
+    Image image = JsonToImageTranslator.toImage(v22Manifest, configurationTemplate);
+    return new LocalImage(image, layers);
+  }
+}

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.cloud.tools.jib.builder.steps;
+
+import com.google.cloud.tools.jib.api.ImageReference;
+import com.google.cloud.tools.jib.configuration.BuildConfiguration;
+import com.google.cloud.tools.jib.docker.DockerClient;
+import com.google.cloud.tools.jib.filesystem.FileOperations;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+
+/** Saves an image from the docker daemon. */
+public class SaveDockerStep implements Callable<Path> {
+
+  private final BuildConfiguration buildConfiguration;
+  private final DockerClient dockerClient;
+
+  SaveDockerStep(BuildConfiguration buildConfiguration, DockerClient dockerClient) {
+    this.buildConfiguration = buildConfiguration;
+    this.dockerClient = dockerClient;
+  }
+
+  @Override
+  public Path call() throws IOException, InterruptedException {
+    Path outputDir = Files.createTempDirectory("jib-docker-save");
+    FileOperations.deleteRecursiveOnExit(outputDir);
+    Path outputPath = outputDir.resolve("out.tar");
+    ImageReference imageReference = buildConfiguration.getBaseImageConfiguration().getImage();
+    dockerClient.save(imageReference, outputPath);
+    return outputPath;
+  }
+}

jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java

@@ -27,7 +27,7 @@
 public class CachedLayer implements Layer {
 
   /** Builds a {@link CachedLayer}. */
-  static class Builder {
+  public static class Builder {
 
     @Nullable private DescriptorDigest layerDigest;
     @Nullable private DescriptorDigest layerDiffId;
@@ -36,22 +36,22 @@ static class Builder {
 
     private Builder() {}
 
-    Builder setLayerDigest(DescriptorDigest layerDigest) {
+    public Builder setLayerDigest(DescriptorDigest layerDigest) {
       this.layerDigest = layerDigest;
       return this;
     }
 
-    Builder setLayerDiffId(DescriptorDigest layerDiffId) {
+    public Builder setLayerDiffId(DescriptorDigest layerDiffId) {
       this.layerDiffId = layerDiffId;
       return this;
     }
 
-    Builder setLayerSize(long layerSize) {
+    public Builder setLayerSize(long layerSize) {
       this.layerSize = layerSize;
       return this;
     }
 
-    Builder setLayerBlob(Blob layerBlob) {
+    public Builder setLayerBlob(Blob layerBlob) {
       this.layerBlob = layerBlob;
       return this;
     }
@@ -60,7 +60,7 @@ boolean hasLayerBlob() {
       return layerBlob != null;
     }
 
-    CachedLayer build() {
+    public CachedLayer build() {
       return new CachedLayer(
           Preconditions.checkNotNull(layerDigest, "layerDigest required"),
           Preconditions.checkNotNull(layerDiffId, "layerDiffId required"),
@@ -74,7 +74,7 @@ CachedLayer build() {
    *
    * @return the new {@link Builder}
    */
-  static Builder builder() {
+  public static Builder builder() {
     return new Builder();
   }
 

jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/FileOperations.java

@@ -17,6 +17,8 @@
 package com.google.cloud.tools.jib.filesystem;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.io.MoreFiles;
+import com.google.common.io.RecursiveDeleteOption;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.channels.Channels;
@@ -85,5 +87,24 @@ public static OutputStream newLockingOutputStream(Path file) throws IOException
     return Channels.newOutputStream(channel);
   }
 
+  /**
+   * Sets up a shutdown hook that tries to delete a file or directory.
+   *
+   * @param path the path to the file or directory
+   */
+  public static void deleteRecursiveOnExit(Path path) {
+    Runtime.getRuntime()
+        .addShutdownHook(
+            new Thread(
+                () -> {
+                  if (Files.exists(path)) {
+                    try {
+                      MoreFiles.deleteRecursively(path, RecursiveDeleteOption.ALLOW_INSECURE);
+                    } catch (IOException ignored) {
+                    }
+                  }
+                }));
+  }
+
   private FileOperations() {}
 }

jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplate.java

@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.google.cloud.tools.jib.api.DescriptorDigest;
 import com.google.cloud.tools.jib.json.JsonTemplate;
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import java.util.ArrayList;
 import java.util.List;
@@ -369,8 +368,11 @@ String getContainerUser() {
     return config.Volumes;
   }
 
-  @VisibleForTesting
-  DescriptorDigest getLayerDiffId(int index) {
+  public DescriptorDigest getLayerDiffId(int index) {
     return rootfs.diff_ids.get(index);
   }
+
+  public int getLayerCount() {
+    return rootfs.diff_ids.size();
+  }
 }