Add offline mode to jib-core

jib-core/CHANGELOG.md

@@ -6,12 +6,14 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - Container configurations in the base image are now propagated when registry uses the old V2 image manifest, schema version 1 (such as Quay) ([#1641](https://github.com/GoogleContainerTools/jib/issues/1641))
+- `Containerizer#setOfflineMode` to retrieve the base image from Jib's cache rather than a container registry ([#718](https://github.com/GoogleContainerTools/jib/issues/718))
 
 ### Changed
 
 ### Fixed
 
 - Labels in the base image are now propagated ([#1643](https://github.com/GoogleContainerTools/jib/issues/1643))
+- Fixed an issue with using OCI base images ([#1683](https://github.com/GoogleContainerTools/jib/issues/1683))
 
 ## 0.9.1
 

jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java

@@ -24,6 +24,7 @@
 import com.google.cloud.tools.jib.registry.LocalRegistry;
 import com.google.cloud.tools.jib.registry.RegistryException;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -32,14 +33,18 @@
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.ClassRule;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
 /** Integration tests for {@link Jib}. */
 public class JibIntegrationTest {
 
   @ClassRule
   public static final LocalRegistry localRegistry = new LocalRegistry(5000, "username", "password");
 
+  @Rule public final TemporaryFolder cacheFolder = new TemporaryFolder();
+
   /**
    * Pulls a built image and attempts to run it.
    *
@@ -109,6 +114,64 @@ public void testScratch()
         inspectOutput.contains("\"Layers\": ["));
   }
 
+  @Test
+  public void testOffline()
+      throws IOException, InterruptedException, InvalidImageReferenceException, ExecutionException,
+          RegistryException, CacheDirectoryCreationException {
+    LocalRegistry tempRegistry = new LocalRegistry(5001);
+    tempRegistry.start();
+    tempRegistry.pullAndPushToLocal("busybox", "busybox");
+    Path cacheDirectory = cacheFolder.getRoot().toPath();
+
+    ImageReference targetImageReferenceOnline =
+        ImageReference.of("localhost:5001", "jib-core", "basic-online");
+    ImageReference targetImageReferenceOffline =
+        ImageReference.of("localhost:5001", "jib-core", "basic-offline");
+
+    JibContainerBuilder jibContainerBuilder =
+        Jib.from("localhost:5001/busybox").setEntrypoint("echo", "Hello World");
+
+    // Should fail since Jib can't build to registry offline
+    try {
+      jibContainerBuilder.containerize(
+          Containerizer.to(RegistryImage.named(targetImageReferenceOffline)).setOfflineMode(true));
+      Assert.fail();
+    } catch (IllegalStateException ex) {
+      Assert.assertEquals("Cannot build to a container registry in offline mode", ex.getMessage());
+    }
+
+    // Should fail since Jib hasn't cached the base image yet
+    try {
+      jibContainerBuilder.containerize(
+          Containerizer.to(DockerDaemonImage.named(targetImageReferenceOffline))
+              .setBaseImageLayersCache(cacheDirectory)
+              .setOfflineMode(true));
+      Assert.fail();
+    } catch (ExecutionException ex) {
+      Assert.assertEquals(
+          "Cannot run Jib in offline mode; localhost:5001/busybox not found in local Jib cache",
+          ex.getCause().getMessage());
+    }
+
+    // Run online to cache the base image
+    jibContainerBuilder.containerize(
+        Containerizer.to(DockerDaemonImage.named(targetImageReferenceOnline))
+            .setBaseImageLayersCache(cacheDirectory)
+            .setAllowInsecureRegistries(true));
+
+    // Run again in offline mode, should succeed this time
+    tempRegistry.stop();
+    jibContainerBuilder.containerize(
+        Containerizer.to(DockerDaemonImage.named(targetImageReferenceOffline))
+            .setBaseImageLayersCache(cacheDirectory)
+            .setOfflineMode(true));
+
+    // Verify output
+    Assert.assertEquals(
+        "Hello World\n",
+        new Command("docker", "run", "--rm", targetImageReferenceOffline.toString()).run());
+  }
+
   /** Ensure that a provided executor is not disposed. */
   @Test
   public void testProvidedExecutorNotDisposed()

jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/LocalRegistry.java

@@ -52,6 +52,16 @@ public LocalRegistry(int port, String username, String password) {
   /** Starts the local registry. */
   @Override
   protected void before() throws IOException, InterruptedException {
+    start();
+  }
+
+  @Override
+  protected void after() {
+    stop();
+  }
+
+  /** Starts the registry */
+  public void start() throws IOException, InterruptedException {
     // Runs the Docker registry.
     ArrayList<String> dockerTokens =
         new ArrayList<>(
@@ -96,8 +106,8 @@ protected void before() throws IOException, InterruptedException {
     waitUntilReady();
   }
 
-  @Override
-  protected void after() {
+  /** Stops the registry. */
+  public void stop() {
     try {
       logout();
       new Command("docker", "stop", containerName).run();

jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java

@@ -39,7 +39,6 @@
 import javax.annotation.Nullable;
 
 /** Configures how to containerize. */
-// TODO: Add tests once JibContainerBuilder#containerize() is added.
 public class Containerizer {
 
   /**
@@ -83,7 +82,7 @@ public static Containerizer to(RegistryImage registryImage) {
                 .pushImage();
 
     return new Containerizer(
-        DESCRIPTION_FOR_DOCKER_REGISTRY, imageConfiguration, stepsRunnerFactory);
+        DESCRIPTION_FOR_DOCKER_REGISTRY, imageConfiguration, stepsRunnerFactory, true);
   }
 
   /**
@@ -110,7 +109,8 @@ public static Containerizer to(DockerDaemonImage dockerDaemonImage) {
                 .buildImage()
                 .loadDocker(dockerClientBuilder.build());
 
-    return new Containerizer(DESCRIPTION_FOR_DOCKER_DAEMON, imageConfiguration, stepsRunnerFactory);
+    return new Containerizer(
+        DESCRIPTION_FOR_DOCKER_DAEMON, imageConfiguration, stepsRunnerFactory, false);
   }
 
   /**
@@ -132,29 +132,34 @@ public static Containerizer to(TarImage tarImage) {
                 .buildImage()
                 .writeTarFile(tarImage.getOutputFile());
 
-    return new Containerizer(DESCRIPTION_FOR_TARBALL, imageConfiguration, stepsRunnerFactory);
+    return new Containerizer(
+        DESCRIPTION_FOR_TARBALL, imageConfiguration, stepsRunnerFactory, false);
   }
 
   private final String description;
   private final ImageConfiguration imageConfiguration;
   private final Function<BuildConfiguration, StepsRunner> stepsRunnerFactory;
+  private final boolean mustBeOnline;
 
   private final Set<String> additionalTags = new HashSet<>();
   @Nullable private ExecutorService executorService;
   private Path baseImageLayersCacheDirectory = DEFAULT_BASE_CACHE_DIRECTORY;
   @Nullable private Path applicationLayersCacheDirectory;
   @Nullable private EventHandlers eventHandlers;
   private boolean allowInsecureRegistries = false;
+  private boolean offline = false;
   private String toolName = DEFAULT_TOOL_NAME;
 
   /** Instantiate with {@link #to}. */
   private Containerizer(
       String description,
       ImageConfiguration imageConfiguration,
-      Function<BuildConfiguration, StepsRunner> stepsRunnerFactory) {
+      Function<BuildConfiguration, StepsRunner> stepsRunnerFactory,
+      boolean mustBeOnline) {
     this.description = description;
     this.imageConfiguration = imageConfiguration;
     this.stepsRunnerFactory = stepsRunnerFactory;
+    this.mustBeOnline = mustBeOnline;
   }
 
   /**
@@ -235,6 +240,22 @@ public Containerizer setAllowInsecureRegistries(boolean allowInsecureRegistries)
     return this;
   }
 
+  /**
+   * Sets whether or not to run the build in offline mode. In offline mode, the base image is
+   * retrieved from the cache instead of pulled from a registry, and the build will fail if the base
+   * image is not in the cache or if the target is an image registry.
+   *
+   * @param offline if {@code true}, the build will run in offline mode
+   * @return this
+   */
+  public Containerizer setOfflineMode(boolean offline) {
+    if (mustBeOnline && offline) {
+      throw new IllegalStateException("Cannot build to a container registry in offline mode");
+    }
+    this.offline = offline;
+    return this;
+  }
+
   /**
    * Sets the name of the tool that is using Jib Core. The tool name is sent as part of the {@code
    * User-Agent} in registry requests and set as the {@code created_by} in the container layer
@@ -283,6 +304,10 @@ boolean getAllowInsecureRegistries() {
     return allowInsecureRegistries;
   }
 
+  boolean isOfflineMode() {
+    return offline;
+  }
+
   String getToolName() {
     return toolName;
   }

jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainerBuilder.java

@@ -476,7 +476,6 @@ JibContainer containerize(
       Containerizer containerizer, Supplier<ExecutorService> defaultExecutorServiceFactory)
       throws IOException, CacheDirectoryCreationException, InterruptedException, RegistryException,
           ExecutionException {
-
     boolean shutdownExecutorService = !containerizer.getExecutorService().isPresent();
     ExecutorService executorService =
         containerizer.getExecutorService().orElseGet(defaultExecutorServiceFactory);
@@ -527,6 +526,7 @@ BuildConfiguration toBuildConfiguration(
         .setContainerConfiguration(containerConfigurationBuilder.build())
         .setLayerConfigurations(layerConfigurations)
         .setAllowInsecureRegistries(containerizer.getAllowInsecureRegistries())
+        .setOffline(containerizer.isOfflineMode())
         .setToolName(containerizer.getToolName())
         .setExecutorService(executorService);
 

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java

@@ -82,6 +82,11 @@ public CachedLayer call() throws IOException, CacheCorruptedException {
       Optional<CachedLayer> optionalCachedLayer = cache.retrieve(layerDigest);
       if (optionalCachedLayer.isPresent()) {
         return optionalCachedLayer.get();
+      } else if (buildConfiguration.isOffline()) {
+        throw new IOException(
+            "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer "
+                + layerDigest
+                + ". You may need to rerun Jib in online mode to re-download the base image layers.");
       }
 
       RegistryClient registryClient =

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java

@@ -23,6 +23,7 @@
 import com.google.cloud.tools.jib.builder.ProgressEventDispatcher;
 import com.google.cloud.tools.jib.builder.TimerEventDispatcher;
 import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.BaseImageWithAuthorization;
+import com.google.cloud.tools.jib.cache.CacheCorruptedException;
 import com.google.cloud.tools.jib.configuration.BuildConfiguration;
 import com.google.cloud.tools.jib.configuration.ImageConfiguration;
 import com.google.cloud.tools.jib.configuration.credentials.Credential;
@@ -32,15 +33,17 @@
 import com.google.cloud.tools.jib.http.Authorizations;
 import com.google.cloud.tools.jib.image.DescriptorDigest;
 import com.google.cloud.tools.jib.image.Image;
+import com.google.cloud.tools.jib.image.ImageReference;
 import com.google.cloud.tools.jib.image.LayerCountMismatchException;
 import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException;
 import com.google.cloud.tools.jib.image.json.BadContainerConfigurationFormatException;
+import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;
 import com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;
 import com.google.cloud.tools.jib.image.json.JsonToImageTranslator;
+import com.google.cloud.tools.jib.image.json.ManifestAndConfig;
 import com.google.cloud.tools.jib.image.json.ManifestTemplate;
 import com.google.cloud.tools.jib.image.json.UnknownManifestFormatException;
 import com.google.cloud.tools.jib.image.json.V21ManifestTemplate;
-import com.google.cloud.tools.jib.image.json.V22ManifestTemplate;
 import com.google.cloud.tools.jib.json.JsonTemplateMapper;
 import com.google.cloud.tools.jib.registry.RegistryAuthenticator;
 import com.google.cloud.tools.jib.registry.RegistryClient;
@@ -51,6 +54,7 @@
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
 import java.io.IOException;
+import java.util.Optional;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import javax.annotation.Nullable;
@@ -106,8 +110,8 @@ public ListenableFuture<BaseImageWithAuthorization> getFuture() {
   @Override
   public BaseImageWithAuthorization call()
       throws IOException, RegistryException, LayerPropertyNotFoundException,
-          LayerCountMismatchException, ExecutionException,
-          BadContainerConfigurationFormatException {
+          LayerCountMismatchException, ExecutionException, BadContainerConfigurationFormatException,
+          CacheCorruptedException {
     // Skip this step if this is a scratch image
     ImageConfiguration baseImageConfiguration = buildConfiguration.getBaseImageConfiguration();
     if (baseImageConfiguration.getImage().isScratch()) {
@@ -126,6 +130,10 @@ public BaseImageWithAuthorization call()
                     + buildConfiguration.getBaseImageConfiguration().getImage()
                     + "..."));
 
+    if (buildConfiguration.isOffline()) {
+      return new BaseImageWithAuthorization(pullBaseImageOffline(), null);
+    }
+
     try (ProgressEventDispatcher progressEventDispatcher =
             progressEventDispatcherFactory.create(
                 BuildStepType.PULL_BASE_IMAGE, "pulling base image manifest", 2);
@@ -227,19 +235,24 @@ private Image pullBaseImage(
     switch (manifestTemplate.getSchemaVersion()) {
       case 1:
         V21ManifestTemplate v21ManifestTemplate = (V21ManifestTemplate) manifestTemplate;
+        buildConfiguration
+            .getBaseImageLayersCache()
+            .writeMetadata(
+                buildConfiguration.getBaseImageConfiguration().getImage(), v21ManifestTemplate);
         return JsonToImageTranslator.toImage(v21ManifestTemplate);
 
       case 2:
-        V22ManifestTemplate v22ManifestTemplate = (V22ManifestTemplate) manifestTemplate;
-        if (v22ManifestTemplate.getContainerConfiguration() == null
-            || v22ManifestTemplate.getContainerConfiguration().getDigest() == null) {
+        BuildableManifestTemplate buildableManifestTemplate =
+            (BuildableManifestTemplate) manifestTemplate;
+        if (buildableManifestTemplate.getContainerConfiguration() == null
+            || buildableManifestTemplate.getContainerConfiguration().getDigest() == null) {
           throw new UnknownManifestFormatException(
-              "Invalid container configuration in Docker V2.2 manifest: \n"
-                  + Blobs.writeToString(JsonTemplateMapper.toBlob(v22ManifestTemplate)));
+              "Invalid container configuration in Docker V2.2/OCI manifest: \n"
+                  + Blobs.writeToString(JsonTemplateMapper.toBlob(buildableManifestTemplate)));
         }
 
         DescriptorDigest containerConfigurationDigest =
-            v22ManifestTemplate.getContainerConfiguration().getDigest();
+            buildableManifestTemplate.getContainerConfiguration().getDigest();
 
         try (ProgressEventDispatcherContainer progressEventDispatcherContainer =
             new ProgressEventDispatcherContainer(
@@ -256,10 +269,49 @@ private Image pullBaseImage(
           ContainerConfigurationTemplate containerConfigurationTemplate =
               JsonTemplateMapper.readJson(
                   containerConfigurationString, ContainerConfigurationTemplate.class);
-          return JsonToImageTranslator.toImage(v22ManifestTemplate, containerConfigurationTemplate);
+          buildConfiguration
+              .getBaseImageLayersCache()
+              .writeMetadata(
+                  buildConfiguration.getBaseImageConfiguration().getImage(),
+                  buildableManifestTemplate,
+                  containerConfigurationTemplate);
+          return JsonToImageTranslator.toImage(
+              buildableManifestTemplate, containerConfigurationTemplate);
         }
     }
 
     throw new IllegalStateException("Unknown manifest schema version");
   }
+
+  /**
+   * Retrieves the cached base image.
+   *
+   * @return the cached image
+   * @throws IOException when an I/O exception occurs
+   * @throws CacheCorruptedException if the cache is corrupted
+   * @throws LayerPropertyNotFoundException if adding image layers fails
+   * @throws BadContainerConfigurationFormatException if the container configuration is in a bad
+   *     format
+   */
+  private Image pullBaseImageOffline()
+      throws IOException, CacheCorruptedException, BadContainerConfigurationFormatException,
+          LayerCountMismatchException {
+    ImageReference baseImage = buildConfiguration.getBaseImageConfiguration().getImage();
+    Optional<ManifestAndConfig> metadata =
+        buildConfiguration.getBaseImageLayersCache().retrieveMetadata(baseImage);
+    if (!metadata.isPresent()) {
+      throw new IOException(
+          "Cannot run Jib in offline mode; " + baseImage + " not found in local Jib cache");
+    }
+
+    ManifestTemplate manifestTemplate = metadata.get().getManifest();
+    if (manifestTemplate instanceof V21ManifestTemplate) {
+      return JsonToImageTranslator.toImage((V21ManifestTemplate) manifestTemplate);
+    }
+
+    ContainerConfigurationTemplate configurationTemplate =
+        metadata.get().getConfig().orElseThrow(IllegalStateException::new);
+    return JsonToImageTranslator.toImage(
+        (BuildableManifestTemplate) manifestTemplate, configurationTemplate);
+  }
 }