From 17f77fc82da12ad20e67a1c0ca97ba32840d92ab Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 7 Jun 2019 18:46:21 -0400 Subject: [PATCH 01/50] wip --- .../cloud/tools/jib/api/Containerizer.java | 56 +-- .../tools/jib/api/JibContainerBuilder.java | 2 +- .../builder/steps/AuthenticatePushStep.java | 38 +- .../BuildAndCacheApplicationLayerStep.java | 41 +- .../jib/builder/steps/BuildImageStep.java | 68 +--- .../jib/builder/steps/CachedLayerAndName.java | 25 ++ .../jib/builder/steps/LoadDockerStep.java | 64 +-- .../steps/PullAndCacheBaseImageLayerStep.java | 31 +- .../PullAndCacheBaseImageLayersStep.java | 69 +--- .../jib/builder/steps/PullBaseImageStep.java | 67 ++- .../tools/jib/builder/steps/PushBlobStep.java | 32 +- .../steps/PushContainerConfigurationStep.java | 66 +-- .../jib/builder/steps/PushImageStep.java | 94 +---- .../jib/builder/steps/PushLayerStep.java | 91 +++++ .../jib/builder/steps/PushLayersStep.java | 116 ------ .../RetrieveRegistryCredentialsStep.java | 26 +- .../tools/jib/builder/steps/StepsRunner.java | 383 ++++++++++-------- .../jib/builder/steps/WriteTarFileStep.java | 53 +-- .../tools/jib/cache/CacheStorageWriter.java | 13 +- .../jib/image/json/ImageToJsonTranslator.java | 1 - .../jib/builder/steps/BuildImageStepTest.java | 2 +- 21 files changed, 506 insertions(+), 832 deletions(-) create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayersStep.java diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java index 96641f4de3..93515e90d0 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java @@ -17,6 +17,7 @@ package com.google.cloud.tools.jib.api; // TODO: Move to com.google.cloud.tools.jib once that package is cleaned up. +import com.google.cloud.tools.jib.builder.steps.BuildResult; import com.google.cloud.tools.jib.builder.steps.StepsRunner; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.configuration.ImageConfiguration; @@ -31,6 +32,7 @@ import java.util.HashSet; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Function; @@ -65,19 +67,7 @@ public static Containerizer to(RegistryImage registryImage) { .setCredentialRetrievers(registryImage.getCredentialRetrievers()) .build(); - Function stepsRunnerFactory = - buildConfiguration -> - StepsRunner.begin(buildConfiguration) - .retrieveTargetRegistryCredentials() - .authenticatePush() - .pullBaseImage() - .pullAndCacheBaseImageLayers() - .pushBaseImageLayers() - .buildAndCacheApplicationLayers() - .buildImage() - .pushContainerConfiguration() - .pushApplicationLayers() - .pushImage(); + Function stepsRunnerFactory = StepsRunner::forRegistryBuild; return new Containerizer( DESCRIPTION_FOR_DOCKER_REGISTRY, imageConfiguration, stepsRunnerFactory, true); @@ -98,17 +88,12 @@ public static Containerizer to(DockerDaemonImage dockerDaemonImage) { dockerClientBuilder.setDockerEnvironment( ImmutableMap.copyOf(dockerDaemonImage.getDockerEnvironment())); - Function stepsRunnerFactory = + Containerizer.Runner buildToDocker = buildConfiguration -> - StepsRunner.begin(buildConfiguration) - .pullBaseImage() - .pullAndCacheBaseImageLayers() - .buildAndCacheApplicationLayers() - .buildImage() - .loadDocker(dockerClientBuilder.build()); + StepsRunner.begin(buildConfiguration).buildToDocker(dockerClientBuilder.build()); return new Containerizer( - DESCRIPTION_FOR_DOCKER_DAEMON, imageConfiguration, stepsRunnerFactory, false); + DESCRIPTION_FOR_DOCKER_DAEMON, imageConfiguration, buildToDocker, false); } /** @@ -121,17 +106,11 @@ public static Containerizer to(TarImage tarImage) { ImageConfiguration imageConfiguration = ImageConfiguration.builder(tarImage.getImageReference()).build(); - Function stepsRunnerFactory = + Containerizer.Runner buildToTar = buildConfiguration -> - StepsRunner.begin(buildConfiguration) - .pullBaseImage() - .pullAndCacheBaseImageLayers() - .buildAndCacheApplicationLayers() - .buildImage() - .writeTarFile(tarImage.getOutputFile()); + StepsRunner.begin(buildConfiguration).buildToTar(tarImage.getOutputFile()); - return new Containerizer( - DESCRIPTION_FOR_TARBALL, imageConfiguration, stepsRunnerFactory, false); + return new Containerizer(DESCRIPTION_FOR_TARBALL, imageConfiguration, buildToTar, false); } private final String description; @@ -148,15 +127,23 @@ public static Containerizer to(TarImage tarImage) { private boolean offline = false; private String toolName = DEFAULT_TOOL_NAME; + public static interface Runner { + + BuildResult build(BuildConfiguration buildConfiguration) + throws InterruptedException, ExecutionException; + } + + private Runner runner; + /** Instantiate with {@link #to}. */ private Containerizer( String description, ImageConfiguration imageConfiguration, - Function stepsRunnerFactory, + Runner runner, boolean mustBeOnline) { this.description = description; this.imageConfiguration = imageConfiguration; - this.stepsRunnerFactory = stepsRunnerFactory; + this.runner = runner; this.mustBeOnline = mustBeOnline; } @@ -318,7 +305,8 @@ ImageConfiguration getImageConfiguration() { return imageConfiguration; } - StepsRunner createStepsRunner(BuildConfiguration buildConfiguration) { - return stepsRunnerFactory.apply(buildConfiguration); + BuildResult run(BuildConfiguration buildConfiguration) + throws InterruptedException, ExecutionException { + return runner.build(buildConfiguration); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainerBuilder.java b/jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainerBuilder.java index 228d4456ce..3e7576e6f3 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainerBuilder.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/api/JibContainerBuilder.java @@ -476,7 +476,7 @@ JibContainer containerize( try (TimerEventDispatcher ignored = new TimerEventDispatcher(eventHandlers, containerizer.getDescription())) { - BuildResult result = containerizer.createStepsRunner(buildConfiguration).run(); + BuildResult result = containerizer.run(buildConfiguration); return new JibContainer(result.getImageDigest(), result.getImageId()); } catch (ExecutionException ex) { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java index 76d2bf5909..2f3d1f6136 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java @@ -18,17 +18,12 @@ import com.google.cloud.tools.jib.api.InsecureRegistryException; import com.google.cloud.tools.jib.api.RegistryException; -import com.google.cloud.tools.jib.async.AsyncDependencies; -import com.google.cloud.tools.jib.async.AsyncStep; -import com.google.cloud.tools.jib.async.NonBlockingSteps; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.configuration.credentials.Credential; import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.registry.RegistryAuthenticator; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; import java.io.IOException; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -40,42 +35,27 @@ * @see https://docs.docker.com/registry/spec/auth/token/ */ -class AuthenticatePushStep implements AsyncStep, Callable { +class AuthenticatePushStep implements Callable { private static final String DESCRIPTION = "Authenticating with push to %s"; private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - - private final RetrieveRegistryCredentialsStep retrieveTargetRegistryCredentialsStep; - - private final ListenableFuture listenableFuture; + private final Credential registryCredentials; AuthenticatePushStep( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, - RetrieveRegistryCredentialsStep retrieveTargetRegistryCredentialsStep) { + Credential targetRegistryCredentials) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.retrieveTargetRegistryCredentialsStep = retrieveTargetRegistryCredentialsStep; - - listenableFuture = - AsyncDependencies.using(listeningExecutorService) - .addStep(retrieveTargetRegistryCredentialsStep) - .whenAllSucceed(this); - } - - @Override - public ListenableFuture getFuture() { - return listenableFuture; + this.registryCredentials = targetRegistryCredentials; } @Override @Nullable - public Authorization call() throws ExecutionException, IOException, RegistryException { - Credential registryCredential = NonBlockingSteps.get(retrieveTargetRegistryCredentialsStep); - + public Authorization call() + throws ExecutionException, IOException, RegistryException, InterruptedException { String registry = buildConfiguration.getTargetImageConfiguration().getImageRegistry(); try (ProgressEventDispatcher ignored = progressEventDispatcherFactory.create("authenticating push to " + registry, 1); @@ -88,15 +68,15 @@ public Authorization call() throws ExecutionException, IOException, RegistryExce .newRegistryClient() .getRegistryAuthenticator(); if (registryAuthenticator != null) { - return registryAuthenticator.authenticatePush(registryCredential); + return registryAuthenticator.authenticatePush(registryCredentials); } } catch (InsecureRegistryException ex) { // Cannot skip certificate validation or use HTTP; fall through. } - return (registryCredential == null || registryCredential.isOAuth2RefreshToken()) + return (registryCredentials == null || registryCredentials.isOAuth2RefreshToken()) ? null : Authorization.fromBasicCredentials( - registryCredential.getUsername(), registryCredential.getPassword()); + registryCredentials.getUsername(), registryCredentials.getPassword()); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java index e3fdea7b04..32971998ca 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java @@ -17,7 +17,6 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.api.LayerConfiguration; -import com.google.cloud.tools.jib.async.AsyncStep; import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; @@ -28,30 +27,27 @@ import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.image.ReproducibleLayerBuilder; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; import java.io.IOException; import java.util.Optional; import java.util.concurrent.Callable; /** Builds and caches application layers. */ -class BuildAndCacheApplicationLayerStep implements AsyncStep, Callable { +class BuildAndCacheApplicationLayerStep implements Callable { - private static final String DESCRIPTION = "Building application layers"; + private static final String DESCRIPTION = "Preparing application layer builders"; /** * Makes a list of {@link BuildAndCacheApplicationLayerStep} for dependencies, resources, and * classes layers. Optionally adds an extra layer if configured to do so. */ static ImmutableList makeList( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory) { int layerCount = buildConfiguration.getLayerConfigurations().size(); try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create( - "setting up to build application layers", layerCount); + "preparing to application layer builders", layerCount); TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { ImmutableList.Builder buildAndCacheApplicationLayerSteps = @@ -64,7 +60,6 @@ static ImmutableList makeList( buildAndCacheApplicationLayerSteps.add( new BuildAndCacheApplicationLayerStep( - listeningExecutorService, buildConfiguration, progressEventDispatcher.newChildProducer(), layerConfiguration.getName(), @@ -79,38 +74,28 @@ static ImmutableList makeList( private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - private final String layerType; + private final String layerName; private final LayerConfiguration layerConfiguration; - private final ListenableFuture listenableFuture; - private BuildAndCacheApplicationLayerStep( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, - String layerType, + String layerName, LayerConfiguration layerConfiguration) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.layerType = layerType; + this.layerName = layerName; this.layerConfiguration = layerConfiguration; - - listenableFuture = listeningExecutorService.submit(this); } @Override - public ListenableFuture getFuture() { - return listenableFuture; - } - - @Override - public CachedLayer call() throws IOException, CacheCorruptedException { - String description = "Building " + layerType + " layer"; + public CachedLayerAndName call() throws IOException, CacheCorruptedException { + String description = "Building " + layerName + " layer"; buildConfiguration.getEventHandlers().dispatch(LogEvent.progress(description + "...")); try (ProgressEventDispatcher ignored = - progressEventDispatcherFactory.create("building " + layerType + " layer", 1); + progressEventDispatcherFactory.create("building " + layerName + " layer", 1); TimerEventDispatcher ignored2 = new TimerEventDispatcher(buildConfiguration.getEventHandlers(), description)) { Cache cache = buildConfiguration.getApplicationLayersCache(); @@ -119,7 +104,7 @@ public CachedLayer call() throws IOException, CacheCorruptedException { Optional optionalCachedLayer = cache.retrieve(layerConfiguration.getLayerEntries()); if (optionalCachedLayer.isPresent()) { - return optionalCachedLayer.get(); + return new CachedLayerAndName(optionalCachedLayer.get(), layerName); } Blob layerBlob = new ReproducibleLayerBuilder(layerConfiguration.getLayerEntries()).build(); @@ -130,11 +115,7 @@ public CachedLayer call() throws IOException, CacheCorruptedException { .getEventHandlers() .dispatch(LogEvent.debug(description + " built " + cachedLayer.getDigest())); - return cachedLayer; + return new CachedLayerAndName(cachedLayer, layerName); } } - - String getLayerType() { - return layerType; - } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java index 3a5b88c2ec..daee8f19d2 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java @@ -17,9 +17,6 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.ProjectInfo; -import com.google.cloud.tools.jib.async.AsyncDependencies; -import com.google.cloud.tools.jib.async.AsyncStep; -import com.google.cloud.tools.jib.async.NonBlockingSteps; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; @@ -28,9 +25,8 @@ import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; import com.google.cloud.tools.jib.image.json.HistoryEntry; +import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; import java.time.Instant; import java.util.List; import java.util.concurrent.Callable; @@ -38,71 +34,44 @@ import javax.annotation.Nullable; /** Builds a model {@link Image}. */ -class BuildImageStep implements AsyncStep>, Callable> { +class BuildImageStep implements Callable { private static final String DESCRIPTION = "Building container configuration"; - private final ListeningExecutorService listeningExecutorService; private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - private final PullBaseImageStep pullBaseImageStep; - private final PullAndCacheBaseImageLayersStep pullAndCacheBaseImageLayersStep; - private final ImmutableList buildAndCacheApplicationLayerSteps; - - private final ListenableFuture> listenableFuture; + private final Image baseImage; + private final List baseImageLayers; + private final List applicationLayers; BuildImageStep( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, - PullBaseImageStep pullBaseImageStep, - PullAndCacheBaseImageLayersStep pullAndCacheBaseImageLayersStep, - ImmutableList buildAndCacheApplicationLayerSteps) { - this.listeningExecutorService = listeningExecutorService; + Image baseImage, + List baseImageLayers, + List applicationLayers) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.pullBaseImageStep = pullBaseImageStep; - this.pullAndCacheBaseImageLayersStep = pullAndCacheBaseImageLayersStep; - this.buildAndCacheApplicationLayerSteps = buildAndCacheApplicationLayerSteps; - - listenableFuture = - AsyncDependencies.using(listeningExecutorService) - .addStep(pullBaseImageStep) - .addStep(pullAndCacheBaseImageLayersStep) - .whenAllSucceed(this); - } - - @Override - public ListenableFuture> getFuture() { - return listenableFuture; + this.baseImage = baseImage; + this.baseImageLayers = baseImageLayers; + this.applicationLayers = applicationLayers; } @Override - public AsyncStep call() throws ExecutionException { - ListenableFuture future = - AsyncDependencies.using(listeningExecutorService) - .addSteps(NonBlockingSteps.get(pullAndCacheBaseImageLayersStep)) - .addSteps(buildAndCacheApplicationLayerSteps) - .whenAllSucceed(this::afterCachedLayerSteps); - return () -> future; - } - - private Image afterCachedLayerSteps() throws ExecutionException, LayerPropertyNotFoundException { + public Image call() + throws ExecutionException, LayerPropertyNotFoundException, InterruptedException { try (ProgressEventDispatcher ignored = progressEventDispatcherFactory.create("building image format", 1); TimerEventDispatcher ignored2 = new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { // Constructs the image. Image.Builder imageBuilder = Image.builder(buildConfiguration.getTargetFormat()); - Image baseImage = NonBlockingSteps.get(pullBaseImageStep).getBaseImage(); ContainerConfiguration containerConfiguration = buildConfiguration.getContainerConfiguration(); // Base image layers - List baseImageLayers = - NonBlockingSteps.get(pullAndCacheBaseImageLayersStep); - for (PullAndCacheBaseImageLayerStep pullAndCacheBaseImageLayerStep : baseImageLayers) { - imageBuilder.addLayer(NonBlockingSteps.get(pullAndCacheBaseImageLayerStep)); + for (CachedLayerAndName baseImageLayer : baseImageLayers) { + imageBuilder.addLayer(baseImageLayer.getCachedLayer()); } // Passthrough config and count non-empty history entries @@ -137,16 +106,15 @@ private Image afterCachedLayerSteps() throws ExecutionException, LayerPropertyNo } // Add built layers/configuration - for (BuildAndCacheApplicationLayerStep buildAndCacheApplicationLayerStep : - buildAndCacheApplicationLayerSteps) { + for (CachedLayerAndName applicationLayer : applicationLayers) { imageBuilder - .addLayer(NonBlockingSteps.get(buildAndCacheApplicationLayerStep)) + .addLayer(applicationLayer.getCachedLayer()) .addHistory( HistoryEntry.builder() .setCreationTimestamp(layerCreationTime) .setAuthor("Jib") .setCreatedBy(buildConfiguration.getToolName() + ":" + ProjectInfo.VERSION) - .setComment(buildAndCacheApplicationLayerStep.getLayerType()) + .setComment(Verify.verifyNotNull(applicationLayer.getName())) .build()); } if (containerConfiguration != null) { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java new file mode 100644 index 0000000000..3e5ec1c9aa --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java @@ -0,0 +1,25 @@ +package com.google.cloud.tools.jib.builder.steps; + +import com.google.cloud.tools.jib.cache.CachedLayer; +import javax.annotation.Nullable; + +/** Simple structure to hold the result pair of {#link CachedLayer} and its name. */ +class CachedLayerAndName { + + private CachedLayer cachedLayer; + @Nullable private String name; + + CachedLayerAndName(CachedLayer cachedLayer, @Nullable String name) { + this.cachedLayer = cachedLayer; + this.name = name; + } + + CachedLayer getCachedLayer() { + return cachedLayer; + } + + @Nullable + String getName() { + return name; + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java index 86bbc6227e..c35030d31e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java @@ -17,92 +17,50 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.api.ImageReference; -import com.google.cloud.tools.jib.async.AsyncDependencies; -import com.google.cloud.tools.jib.async.AsyncStep; -import com.google.cloud.tools.jib.async.NonBlockingSteps; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.DockerClient; import com.google.cloud.tools.jib.docker.ImageTarball; +import com.google.cloud.tools.jib.event.EventHandlers; import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.image.Image; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; import java.io.IOException; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; /** Adds image layers to a tarball and loads into Docker daemon. */ -class LoadDockerStep implements AsyncStep, Callable { +class LoadDockerStep implements Callable { - private final ListeningExecutorService listeningExecutorService; private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; private final DockerClient dockerClient; - - private final PullAndCacheBaseImageLayersStep pullAndCacheBaseImageLayersStep; - private final ImmutableList buildAndCacheApplicationLayerSteps; - private final BuildImageStep buildImageStep; - - private final ListenableFuture listenableFuture; + private final Image builtImage; LoadDockerStep( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, DockerClient dockerClient, - PullAndCacheBaseImageLayersStep pullAndCacheBaseImageLayersStep, - ImmutableList buildAndCacheApplicationLayerSteps, - BuildImageStep buildImageStep) { - this.listeningExecutorService = listeningExecutorService; + Image builtImage) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.dockerClient = dockerClient; - this.pullAndCacheBaseImageLayersStep = pullAndCacheBaseImageLayersStep; - this.buildAndCacheApplicationLayerSteps = buildAndCacheApplicationLayerSteps; - this.buildImageStep = buildImageStep; - - listenableFuture = - AsyncDependencies.using(listeningExecutorService) - .addStep(pullAndCacheBaseImageLayersStep) - .addStep(buildImageStep) - .whenAllSucceed(this); - } - - @Override - public ListenableFuture getFuture() { - return listenableFuture; + this.builtImage = builtImage; } @Override - public BuildResult call() throws ExecutionException, InterruptedException { - return AsyncDependencies.using(listeningExecutorService) - .addSteps(NonBlockingSteps.get(pullAndCacheBaseImageLayersStep)) - .addSteps(buildAndCacheApplicationLayerSteps) - .addStep(NonBlockingSteps.get(buildImageStep)) - .whenAllSucceed(this::afterPushBaseImageLayerFuturesFuture) - .get(); - } - - private BuildResult afterPushBaseImageLayerFuturesFuture() - throws ExecutionException, InterruptedException, IOException { - buildConfiguration - .getEventHandlers() - .dispatch(LogEvent.progress("Loading to Docker daemon...")); + public BuildResult call() throws ExecutionException, InterruptedException, IOException { + EventHandlers eventHandlers = buildConfiguration.getEventHandlers(); + eventHandlers.dispatch(LogEvent.progress("Loading to Docker daemon...")); try (ProgressEventDispatcher ignored = progressEventDispatcherFactory.create("loading to Docker daemon", 1)) { - Image image = NonBlockingSteps.get(NonBlockingSteps.get(buildImageStep)); ImageReference targetImageReference = buildConfiguration.getTargetImageConfiguration().getImage(); // Load the image to docker daemon. - buildConfiguration - .getEventHandlers() - .dispatch( - LogEvent.debug(dockerClient.load(new ImageTarball(image, targetImageReference)))); + eventHandlers.dispatch( + LogEvent.debug(dockerClient.load(new ImageTarball(builtImage, targetImageReference)))); // Tags the image with all the additional tags, skipping the one 'docker load' already loaded. for (String tag : buildConfiguration.getAllTargetImageTags()) { @@ -113,7 +71,7 @@ private BuildResult afterPushBaseImageLayerFuturesFuture() dockerClient.tag(targetImageReference, targetImageReference.withTag(tag)); } - return BuildResult.fromImage(image, buildConfiguration.getTargetFormat()); + return BuildResult.fromImage(builtImage, buildConfiguration.getTargetFormat()); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 9f5913cecd..7fbf042b75 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -17,7 +17,6 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.api.DescriptorDigest; -import com.google.cloud.tools.jib.async.AsyncStep; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.cache.Cache; @@ -26,15 +25,13 @@ import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.registry.RegistryClient; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; import java.io.IOException; import java.util.Optional; import java.util.concurrent.Callable; import javax.annotation.Nullable; /** Pulls and caches a single base image layer. */ -class PullAndCacheBaseImageLayerStep implements AsyncStep, Callable { +class PullAndCacheBaseImageLayerStep implements Callable { private static final String DESCRIPTION = "Pulling base image layer %s"; @@ -44,10 +41,7 @@ class PullAndCacheBaseImageLayerStep implements AsyncStep, Callable private final DescriptorDigest layerDigest; private final @Nullable Authorization pullAuthorization; - private final ListenableFuture listenableFuture; - PullAndCacheBaseImageLayerStep( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, DescriptorDigest layerDigest, @@ -56,17 +50,10 @@ class PullAndCacheBaseImageLayerStep implements AsyncStep, Callable this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.layerDigest = layerDigest; this.pullAuthorization = pullAuthorization; - - listenableFuture = listeningExecutorService.submit(this); - } - - @Override - public ListenableFuture getFuture() { - return listenableFuture; } @Override - public CachedLayer call() throws IOException, CacheCorruptedException { + public CachedLayerAndName call() throws IOException, CacheCorruptedException { try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create("checking base image layer " + layerDigest, 1); TimerEventDispatcher ignored = @@ -77,7 +64,7 @@ public CachedLayer call() throws IOException, CacheCorruptedException { // Checks if the layer already exists in the cache. Optional optionalCachedLayer = cache.retrieve(layerDigest); if (optionalCachedLayer.isPresent()) { - return optionalCachedLayer.get(); + return new CachedLayerAndName(optionalCachedLayer.get(), null); } else if (buildConfiguration.isOffline()) { throw new IOException( "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " @@ -95,11 +82,13 @@ public CachedLayer call() throws IOException, CacheCorruptedException { new ThrottledProgressEventDispatcherWrapper( progressEventDispatcher.newChildProducer(), "pulling base image layer " + layerDigest)) { - return cache.writeCompressedLayer( - registryClient.pullBlob( - layerDigest, - progressEventDispatcherWrapper::setProgressTarget, - progressEventDispatcherWrapper::dispatchProgress)); + CachedLayer cachedLayer = + cache.writeCompressedLayer( + registryClient.pullBlob( + layerDigest, + progressEventDispatcherWrapper::setProgressTarget, + progressEventDispatcherWrapper::dispatchProgress)); + return new CachedLayerAndName(cachedLayer, null); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java index 18a499e10a..1ade0d839e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java @@ -16,81 +16,44 @@ package com.google.cloud.tools.jib.builder.steps; -import com.google.cloud.tools.jib.async.AsyncDependencies; -import com.google.cloud.tools.jib.async.AsyncStep; -import com.google.cloud.tools.jib.async.NonBlockingSteps; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.BaseImageWithAuthorization; +import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.image.Layer; -import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; +import java.util.ArrayList; +import java.util.List; /** Pulls and caches the base image layers. */ -class PullAndCacheBaseImageLayersStep - implements AsyncStep>, - Callable> { +// TODO: following the same pattern as "BuildAndCacheApplicationLayerStep", move the sole "makeList" +// static into "PullAndCacheBaseImageLayerStep" and remove this class. +class PullAndCacheBaseImageLayersStep { - private static final String DESCRIPTION = "Setting up base image caching"; + private static final String DESCRIPTION = "Preparing base image layer pullers"; - private final BuildConfiguration buildConfiguration; - private final ListeningExecutorService listeningExecutorService; - private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - - private final PullBaseImageStep pullBaseImageStep; - - private final ListenableFuture> listenableFuture; - - PullAndCacheBaseImageLayersStep( - ListeningExecutorService listeningExecutorService, + static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, - PullBaseImageStep pullBaseImageStep) { - this.listeningExecutorService = listeningExecutorService; - this.buildConfiguration = buildConfiguration; - this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.pullBaseImageStep = pullBaseImageStep; - - listenableFuture = - AsyncDependencies.using(listeningExecutorService) - .addStep(pullBaseImageStep) - .whenAllSucceed(this); - } - - @Override - public ListenableFuture> getFuture() { - return listenableFuture; - } - - @Override - public ImmutableList call() - throws ExecutionException, LayerPropertyNotFoundException { - BaseImageWithAuthorization pullBaseImageStepResult = NonBlockingSteps.get(pullBaseImageStep); - ImmutableList baseImageLayers = pullBaseImageStepResult.getBaseImage().getLayers(); + ImageAndAuthorization baseImageAndAuth) { + ImmutableList baseImageLayers = baseImageAndAuth.getImage().getLayers(); try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create( - "checking base image layers", baseImageLayers.size()); + "preparing base image layer pullers", baseImageLayers.size()); TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { - ImmutableList.Builder pullAndCacheBaseImageLayerStepsBuilder = - ImmutableList.builderWithExpectedSize(baseImageLayers.size()); + + List layerPullers = new ArrayList<>(); for (Layer layer : baseImageLayers) { - pullAndCacheBaseImageLayerStepsBuilder.add( + layerPullers.add( new PullAndCacheBaseImageLayerStep( - listeningExecutorService, buildConfiguration, progressEventDispatcher.newChildProducer(), layer.getBlobDescriptor().getDigest(), - pullBaseImageStepResult.getBaseImageAuthorization())); + baseImageAndAuth.getAuthorization())); } - - return pullAndCacheBaseImageLayerStepsBuilder.build(); + return ImmutableList.copyOf(layerPullers); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index 1dc151a436..2a9eede849 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -21,12 +21,10 @@ import com.google.cloud.tools.jib.api.InsecureRegistryException; import com.google.cloud.tools.jib.api.RegistryException; import com.google.cloud.tools.jib.api.RegistryUnauthorizedException; -import com.google.cloud.tools.jib.async.AsyncStep; -import com.google.cloud.tools.jib.async.NonBlockingSteps; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.BaseImageWithAuthorization; +import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; import com.google.cloud.tools.jib.cache.CacheCorruptedException; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.configuration.ImageConfiguration; @@ -49,10 +47,9 @@ import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.cloud.tools.jib.registry.RegistryAuthenticator; import com.google.cloud.tools.jib.registry.RegistryClient; +import com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException; import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import java.io.IOException; import java.util.Optional; import java.util.concurrent.Callable; @@ -60,64 +57,61 @@ import javax.annotation.Nullable; /** Pulls the base image manifest. */ -class PullBaseImageStep - implements AsyncStep, Callable { +class PullBaseImageStep implements Callable { private static final String DESCRIPTION = "Pulling base image manifest"; /** Structure for the result returned by this step. */ - static class BaseImageWithAuthorization { + static class ImageAndAuthorization { - private final Image baseImage; - private final @Nullable Authorization baseImageAuthorization; + private final Image image; + private final @Nullable Authorization authorization; @VisibleForTesting - BaseImageWithAuthorization(Image baseImage, @Nullable Authorization baseImageAuthorization) { - this.baseImage = baseImage; - this.baseImageAuthorization = baseImageAuthorization; + ImageAndAuthorization(Image image, @Nullable Authorization authorization) { + this.image = image; + this.authorization = authorization; } - Image getBaseImage() { - return baseImage; + Image getImage() { + return image; } @Nullable - Authorization getBaseImageAuthorization() { - return baseImageAuthorization; + Authorization getAuthorization() { + return authorization; } } private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - private final ListenableFuture listenableFuture; - PullBaseImageStep( ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; - - listenableFuture = listeningExecutorService.submit(this); } - @Override - public ListenableFuture getFuture() { - return listenableFuture; + PullBaseImageStep( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory) { + this.buildConfiguration = buildConfiguration; + this.progressEventDispatcherFactory = progressEventDispatcherFactory; } @Override - public BaseImageWithAuthorization call() + public ImageAndAuthorization call() throws IOException, RegistryException, LayerPropertyNotFoundException, LayerCountMismatchException, ExecutionException, BadContainerConfigurationFormatException, - CacheCorruptedException { + CacheCorruptedException, CredentialRetrievalException { EventHandlers eventHandlers = buildConfiguration.getEventHandlers(); // Skip this step if this is a scratch image ImageConfiguration baseImageConfiguration = buildConfiguration.getBaseImageConfiguration(); if (baseImageConfiguration.getImage().isScratch()) { eventHandlers.dispatch(LogEvent.progress("Getting scratch base image...")); - return new BaseImageWithAuthorization( + return new ImageAndAuthorization( Image.builder(buildConfiguration.getTargetFormat()).build(), null); } @@ -128,7 +122,7 @@ public BaseImageWithAuthorization call() + "...")); if (buildConfiguration.isOffline()) { - return new BaseImageWithAuthorization(pullBaseImageOffline(), null); + return new ImageAndAuthorization(pullBaseImageOffline(), null); } try (ProgressEventDispatcher progressEventDispatcher = @@ -137,7 +131,7 @@ public BaseImageWithAuthorization call() new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { // First, try with no credentials. try { - return new BaseImageWithAuthorization(pullBaseImage(null, progressEventDispatcher), null); + return new ImageAndAuthorization(pullBaseImage(null, progressEventDispatcher), null); } catch (RegistryUnauthorizedException ex) { eventHandlers.dispatch( @@ -148,15 +142,12 @@ public BaseImageWithAuthorization call() // If failed, then, retrieve base registry credentials and try with retrieved credentials. // TODO: Refactor the logic in RetrieveRegistryCredentialsStep out to - // registry.credentials.RegistryCredentialsRetriever to avoid this direct executor hack. - ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService(); - RetrieveRegistryCredentialsStep retrieveBaseRegistryCredentialsStep = + // registry.credentials.RegistryCredentialsRetriever. + Credential registryCredential = RetrieveRegistryCredentialsStep.forBaseImage( - directExecutorService, - buildConfiguration, - progressEventDispatcher.newChildProducer()); + buildConfiguration, progressEventDispatcher.newChildProducer()) + .call(); - Credential registryCredential = NonBlockingSteps.get(retrieveBaseRegistryCredentialsStep); Authorization registryAuthorization = registryCredential == null || registryCredential.isOAuth2RefreshToken() ? null @@ -164,7 +155,7 @@ public BaseImageWithAuthorization call() registryCredential.getUsername(), registryCredential.getPassword()); try { - return new BaseImageWithAuthorization( + return new ImageAndAuthorization( pullBaseImage(registryAuthorization, progressEventDispatcher), registryAuthorization); } catch (RegistryUnauthorizedException registryUnauthorizedException) { @@ -180,7 +171,7 @@ public BaseImageWithAuthorization call() Authorization pullAuthorization = registryAuthenticator.authenticatePull(registryCredential); - return new BaseImageWithAuthorization( + return new ImageAndAuthorization( pullBaseImage(pullAuthorization, progressEventDispatcher), pullAuthorization); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java index cd41886673..ea27fb7cdd 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java @@ -17,9 +17,6 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.api.RegistryException; -import com.google.cloud.tools.jib.async.AsyncDependencies; -import com.google.cloud.tools.jib.async.AsyncStep; -import com.google.cloud.tools.jib.async.NonBlockingSteps; import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; @@ -27,53 +24,38 @@ import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer; +import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.registry.RegistryClient; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; import java.io.IOException; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; /** Pushes a BLOB to the target registry. */ -class PushBlobStep implements AsyncStep, Callable { +class PushBlobStep implements Callable { private static final String DESCRIPTION = "Pushing BLOB "; private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDipatcherFactory; - private final AuthenticatePushStep authenticatePushStep; + private final Authorization authorization; private final BlobDescriptor blobDescriptor; private final Blob blob; - private final ListenableFuture listenableFuture; - PushBlobStep( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDipatcherFactory, - AuthenticatePushStep authenticatePushStep, + Authorization authorization, BlobDescriptor blobDescriptor, Blob blob) { this.buildConfiguration = buildConfiguration; this.progressEventDipatcherFactory = progressEventDipatcherFactory; - this.authenticatePushStep = authenticatePushStep; + this.authorization = authorization; this.blobDescriptor = blobDescriptor; this.blob = blob; - - listenableFuture = - AsyncDependencies.using(listeningExecutorService) - .addStep(authenticatePushStep) - .whenAllSucceed(this); - } - - @Override - public ListenableFuture getFuture() { - return listenableFuture; } @Override - public BlobDescriptor call() throws IOException, RegistryException, ExecutionException { + public BlobDescriptor call() throws IOException, RegistryException { try (ProgressEventDispatcher progressEventDispatcher = progressEventDipatcherFactory.create( "pushing blob " + blobDescriptor.getDigest(), blobDescriptor.getSize()); @@ -85,7 +67,7 @@ public BlobDescriptor call() throws IOException, RegistryException, ExecutionExc RegistryClient registryClient = buildConfiguration .newTargetImageRegistryClientFactory() - .setAuthorization(NonBlockingSteps.get(authenticatePushStep)) + .setAuthorization(authorization) .newRegistryClient(); // check if the BLOB is available diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java index 8ffbe1345a..ef74f438d3 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java @@ -16,90 +16,58 @@ package com.google.cloud.tools.jib.builder.steps; -import com.google.cloud.tools.jib.async.AsyncDependencies; -import com.google.cloud.tools.jib.async.AsyncStep; -import com.google.cloud.tools.jib.async.NonBlockingSteps; +import com.google.cloud.tools.jib.api.RegistryException; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.hash.Digests; +import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; import com.google.cloud.tools.jib.json.JsonTemplate; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; import java.io.IOException; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; /** Pushes the container configuration. */ -class PushContainerConfigurationStep - implements AsyncStep>, Callable> { +class PushContainerConfigurationStep implements Callable { private static final String DESCRIPTION = "Pushing container configuration"; private final BuildConfiguration buildConfiguration; - private final ListeningExecutorService listeningExecutorService; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - private final AuthenticatePushStep authenticatePushStep; - private final BuildImageStep buildImageStep; - - private final ListenableFuture> listenableFuture; + private final Authorization pushAuthorization; + private final Image builtImage; PushContainerConfigurationStep( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, - AuthenticatePushStep authenticatePushStep, - BuildImageStep buildImageStep) { - this.listeningExecutorService = listeningExecutorService; + Authorization authenticatePushStep, + Image builtImage) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.authenticatePushStep = authenticatePushStep; - this.buildImageStep = buildImageStep; - - listenableFuture = - AsyncDependencies.using(listeningExecutorService) - .addStep(buildImageStep) - .whenAllSucceed(this); + this.pushAuthorization = authenticatePushStep; + this.builtImage = builtImage; } @Override - public ListenableFuture> getFuture() { - return listenableFuture; - } - - @Override - public AsyncStep call() throws ExecutionException { - ListenableFuture pushBlobStepFuture = - AsyncDependencies.using(listeningExecutorService) - .addStep(authenticatePushStep) - .addStep(NonBlockingSteps.get(buildImageStep)) - .whenAllSucceed(this::afterBuildConfigurationFutureFuture); - return () -> pushBlobStepFuture; - } - - private PushBlobStep afterBuildConfigurationFutureFuture() - throws ExecutionException, IOException { + public BlobDescriptor call() throws IOException, RegistryException { try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create("pushing container configuration", 1); TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { - Image image = NonBlockingSteps.get(NonBlockingSteps.get(buildImageStep)); JsonTemplate containerConfiguration = - new ImageToJsonTranslator(image).getContainerConfiguration(); - BlobDescriptor blobDescriptor = Digests.computeDigest(containerConfiguration); + new ImageToJsonTranslator(builtImage).getContainerConfiguration(); return new PushBlobStep( - listeningExecutorService, - buildConfiguration, - progressEventDispatcher.newChildProducer(), - authenticatePushStep, - blobDescriptor, - Blobs.from(containerConfiguration)); + buildConfiguration, + progressEventDispatcher.newChildProducer(), + pushAuthorization, + Digests.computeDigest(containerConfiguration), + Blobs.from(containerConfiguration)) + .call(); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java index bdca799169..5bc51dd78b 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java @@ -16,16 +16,13 @@ package com.google.cloud.tools.jib.builder.steps; -import com.google.cloud.tools.jib.api.DescriptorDigest; -import com.google.cloud.tools.jib.async.AsyncDependencies; -import com.google.cloud.tools.jib.async.AsyncStep; -import com.google.cloud.tools.jib.async.NonBlockingSteps; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.events.LogEvent; -import com.google.cloud.tools.jib.hash.Digests; +import com.google.cloud.tools.jib.http.Authorization; +import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; import com.google.cloud.tools.jib.registry.RegistryClient; @@ -40,84 +37,37 @@ import java.util.concurrent.ExecutionException; /** Pushes the final image. Outputs the pushed image digest. */ -class PushImageStep implements AsyncStep, Callable { +class PushImageStep implements Callable { private static final String DESCRIPTION = "Pushing new image"; private final BuildConfiguration buildConfiguration; - private final ListeningExecutorService listeningExecutorService; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - private final AuthenticatePushStep authenticatePushStep; - - private final PushLayersStep pushBaseImageLayersStep; - private final PushLayersStep pushApplicationLayersStep; - private final PushContainerConfigurationStep pushContainerConfigurationStep; - private final BuildImageStep buildImageStep; + private final Authorization pushAuthorization; + private final BlobDescriptor containerConfigurationDigestAndSize; + private final Image builtImage; - private final ListenableFuture listenableFuture; + private final ListeningExecutorService listeningExecutorService; + // TODO: remove listeningExecutorService like other siblings PushImageStep( ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, - AuthenticatePushStep authenticatePushStep, - PushLayersStep pushBaseImageLayersStep, - PushLayersStep pushApplicationLayersStep, - PushContainerConfigurationStep pushContainerConfigurationStep, - BuildImageStep buildImageStep) { + Authorization pushAuthorization, + BlobDescriptor containerConfigurationDigestAndSize, + Image builtImage) { this.listeningExecutorService = listeningExecutorService; this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.authenticatePushStep = authenticatePushStep; - this.pushBaseImageLayersStep = pushBaseImageLayersStep; - this.pushApplicationLayersStep = pushApplicationLayersStep; - this.pushContainerConfigurationStep = pushContainerConfigurationStep; - this.buildImageStep = buildImageStep; - - listenableFuture = - AsyncDependencies.using(listeningExecutorService) - .addStep(pushBaseImageLayersStep) - .addStep(pushApplicationLayersStep) - .addStep(pushContainerConfigurationStep) - .whenAllSucceed(this); - } - - @Override - public ListenableFuture getFuture() { - return listenableFuture; + this.pushAuthorization = pushAuthorization; + this.containerConfigurationDigestAndSize = containerConfigurationDigestAndSize; + this.builtImage = builtImage; } @Override - public BuildResult call() throws ExecutionException, InterruptedException { - return AsyncDependencies.using(listeningExecutorService) - .addStep(authenticatePushStep) - .addSteps(NonBlockingSteps.get(pushBaseImageLayersStep)) - .addSteps(NonBlockingSteps.get(pushApplicationLayersStep)) - .addStep(NonBlockingSteps.get(pushContainerConfigurationStep)) - .addStep(NonBlockingSteps.get(buildImageStep)) - .whenAllSucceed(this::afterPushSteps) - .get(); - } - - private BuildResult afterPushSteps() throws ExecutionException, InterruptedException { - AsyncDependencies dependencies = AsyncDependencies.using(listeningExecutorService); - for (AsyncStep pushBaseImageLayerStep : - NonBlockingSteps.get(pushBaseImageLayersStep)) { - dependencies.addStep(NonBlockingSteps.get(pushBaseImageLayerStep)); - } - for (AsyncStep pushApplicationLayerStep : - NonBlockingSteps.get(pushApplicationLayersStep)) { - dependencies.addStep(NonBlockingSteps.get(pushApplicationLayerStep)); - } - return dependencies - .addStep(NonBlockingSteps.get(NonBlockingSteps.get(pushContainerConfigurationStep))) - .whenAllSucceed(this::afterAllPushed) - .get(); - } - - private BuildResult afterAllPushed() - throws ExecutionException, IOException, InterruptedException { + public BuildResult call() throws IOException, InterruptedException, ExecutionException { ImmutableSet targetImageTags = buildConfiguration.getAllTargetImageTags(); ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create("pushing image manifest", targetImageTags.size()); @@ -127,20 +77,16 @@ private BuildResult afterAllPushed() RegistryClient registryClient = buildConfiguration .newTargetImageRegistryClientFactory() - .setAuthorization(NonBlockingSteps.get(authenticatePushStep)) + .setAuthorization(pushAuthorization) .newRegistryClient(); // Constructs the image. - ImageToJsonTranslator imageToJsonTranslator = - new ImageToJsonTranslator(NonBlockingSteps.get(NonBlockingSteps.get(buildImageStep))); + ImageToJsonTranslator imageToJsonTranslator = new ImageToJsonTranslator(builtImage); // Gets the image manifest to push. - BlobDescriptor containerConfigurationBlobDescriptor = - NonBlockingSteps.get( - NonBlockingSteps.get(NonBlockingSteps.get(pushContainerConfigurationStep))); BuildableManifestTemplate manifestTemplate = imageToJsonTranslator.getManifestTemplate( - buildConfiguration.getTargetFormat(), containerConfigurationBlobDescriptor); + buildConfiguration.getTargetFormat(), containerConfigurationDigestAndSize); // Pushes to all target image tags. List> pushAllTagsFutures = new ArrayList<>(); @@ -161,9 +107,7 @@ private BuildResult afterAllPushed() })); } - DescriptorDigest imageDigest = Digests.computeJsonDigest(manifestTemplate); - DescriptorDigest imageId = containerConfigurationBlobDescriptor.getDigest(); - BuildResult result = new BuildResult(imageDigest, imageId); + BuildResult result = BuildResult.fromImage(builtImage, buildConfiguration.getTargetFormat()); return Futures.whenAllSucceed(pushAllTagsFutures) .call( diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java new file mode 100644 index 0000000000..26e9b552b8 --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java @@ -0,0 +1,91 @@ +/* + * Copyright 2018 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.builder.steps; + +import com.google.cloud.tools.jib.api.RegistryException; +import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; +import com.google.cloud.tools.jib.builder.TimerEventDispatcher; +import com.google.cloud.tools.jib.cache.CachedLayer; +import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.http.Authorization; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +class PushLayerStep implements Callable { + + private static final String DESCRIPTION = "Preparing application layer pushers"; + + private final BuildConfiguration buildConfiguration; + private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; + + private final Authorization pushAuthorization; + private final Future cachedLayerAndName; + + static ImmutableList makeList( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, + Authorization pushAuthorization, + List> cachedLayers) { + try (TimerEventDispatcher ignored = + new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION); + ProgressEventDispatcher progressEventDispatcher = + progressEventDispatcherFactory.create( + "Preparing application layer pushers", cachedLayers.size())) { + + // Constructs a PushBlobStep for each layer. + List blobPushers = new ArrayList<>(); + for (Future layer : cachedLayers) { + ProgressEventDispatcher.Factory childProgressProducer = + progressEventDispatcher.newChildProducer(); + blobPushers.add( + new PushLayerStep(buildConfiguration, childProgressProducer, pushAuthorization, layer)); + } + + return ImmutableList.copyOf(blobPushers); + } + } + + PushLayerStep( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, + Authorization pushAuthorization, + Future cachedLayerAndName) { + this.buildConfiguration = buildConfiguration; + this.progressEventDispatcherFactory = progressEventDispatcherFactory; + this.pushAuthorization = pushAuthorization; + this.cachedLayerAndName = cachedLayerAndName; + } + + @Override + public BlobDescriptor call() + throws IOException, RegistryException, ExecutionException, InterruptedException { + CachedLayer layer = cachedLayerAndName.get().getCachedLayer(); + return new PushBlobStep( + buildConfiguration, + progressEventDispatcherFactory, + pushAuthorization, + new BlobDescriptor(layer.getSize(), layer.getDigest()), + layer.getBlob()) + .call(); + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayersStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayersStep.java deleted file mode 100644 index 299b6c0bd9..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayersStep.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.builder.steps; - -import com.google.cloud.tools.jib.async.AsyncDependencies; -import com.google.cloud.tools.jib.async.AsyncStep; -import com.google.cloud.tools.jib.async.NonBlockingSteps; -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; -import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.cache.CachedLayer; -import com.google.cloud.tools.jib.configuration.BuildConfiguration; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; - -class PushLayersStep - implements AsyncStep>>, - Callable>> { - - private static final String DESCRIPTION = "Setting up to push layers"; - - private final BuildConfiguration buildConfiguration; - private final ListeningExecutorService listeningExecutorService; - private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - - private final AuthenticatePushStep authenticatePushStep; - private final AsyncStep>> - cachedLayerStep; - - private final ListenableFuture>> listenableFuture; - - PushLayersStep( - ListeningExecutorService listeningExecutorService, - BuildConfiguration buildConfiguration, - ProgressEventDispatcher.Factory progressEventDispatcherFactory, - AuthenticatePushStep authenticatePushStep, - AsyncStep>> - cachedLayerStep) { - this.listeningExecutorService = listeningExecutorService; - this.buildConfiguration = buildConfiguration; - this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.authenticatePushStep = authenticatePushStep; - this.cachedLayerStep = cachedLayerStep; - - listenableFuture = - AsyncDependencies.using(listeningExecutorService) - .addStep(cachedLayerStep) - .whenAllSucceed(this); - } - - @Override - public ListenableFuture>> getFuture() { - return listenableFuture; - } - - @Override - public ImmutableList> call() throws ExecutionException { - try (TimerEventDispatcher ignored = - new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { - ImmutableList> cachedLayers = - NonBlockingSteps.get(cachedLayerStep); - - try (ProgressEventDispatcher progressEventDispatcher = - progressEventDispatcherFactory.create("setting up to push layers", cachedLayers.size())) { - // Constructs a PushBlobStep for each layer. - ImmutableList.Builder> pushBlobStepsBuilder = - ImmutableList.builder(); - for (AsyncStep cachedLayerStep : cachedLayers) { - ProgressEventDispatcher.Factory childProgressEventDispatcherFactory = - progressEventDispatcher.newChildProducer(); - ListenableFuture pushBlobStepFuture = - Futures.whenAllSucceed(cachedLayerStep.getFuture()) - .call( - () -> makePushBlobStep(cachedLayerStep, childProgressEventDispatcherFactory), - listeningExecutorService); - pushBlobStepsBuilder.add(() -> pushBlobStepFuture); - } - - return pushBlobStepsBuilder.build(); - } - } - } - - private PushBlobStep makePushBlobStep( - AsyncStep cachedLayerStep, - ProgressEventDispatcher.Factory progressEventDispatcherFactory) - throws ExecutionException { - CachedLayer cachedLayer = NonBlockingSteps.get(cachedLayerStep); - - return new PushBlobStep( - listeningExecutorService, - buildConfiguration, - progressEventDispatcherFactory, - authenticatePushStep, - new BlobDescriptor(cachedLayer.getSize(), cachedLayer.getDigest()), - cachedLayer.getBlob()); - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStep.java index 052abe5c20..66cda40af6 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStep.java @@ -16,7 +16,6 @@ package com.google.cloud.tools.jib.builder.steps; -import com.google.cloud.tools.jib.async.AsyncStep; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; @@ -26,26 +25,18 @@ import com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; import java.util.Optional; import java.util.concurrent.Callable; import javax.annotation.Nullable; /** Attempts to retrieve registry credentials. */ -class RetrieveRegistryCredentialsStep implements AsyncStep, Callable { - - private static String makeDescription(String registry) { - return "Retrieving registry credentials for " + registry; - } +class RetrieveRegistryCredentialsStep implements Callable { /** Retrieves credentials for the base image. */ static RetrieveRegistryCredentialsStep forBaseImage( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory) { return new RetrieveRegistryCredentialsStep( - listeningExecutorService, buildConfiguration, progressEventDispatcherFactory, buildConfiguration.getBaseImageConfiguration().getImageRegistry(), @@ -54,11 +45,9 @@ static RetrieveRegistryCredentialsStep forBaseImage( /** Retrieves credentials for the target image. */ static RetrieveRegistryCredentialsStep forTargetImage( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory) { return new RetrieveRegistryCredentialsStep( - listeningExecutorService, buildConfiguration, progressEventDispatcherFactory, buildConfiguration.getTargetImageConfiguration().getImageRegistry(), @@ -71,11 +60,8 @@ static RetrieveRegistryCredentialsStep forTargetImage( private final String registry; private final ImmutableList credentialRetrievers; - private final ListenableFuture listenableFuture; - @VisibleForTesting RetrieveRegistryCredentialsStep( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, String registry, @@ -84,20 +70,12 @@ static RetrieveRegistryCredentialsStep forTargetImage( this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.registry = registry; this.credentialRetrievers = credentialRetrievers; - - listenableFuture = listeningExecutorService.submit(this); - } - - @Override - public ListenableFuture getFuture() { - return listenableFuture; } @Override @Nullable public Credential call() throws CredentialRetrievalException { - String description = makeDescription(registry); - + String description = "Retrieving registry credentials for " + registry; buildConfiguration.getEventHandlers().dispatch(LogEvent.progress(description + "...")); try (ProgressEventDispatcher ignored = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index fff5d935eb..352ac1ed41 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -16,19 +16,31 @@ package com.google.cloud.tools.jib.builder.steps; -import com.google.cloud.tools.jib.async.AsyncStep; -import com.google.cloud.tools.jib.async.AsyncSteps; +import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; +import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.credentials.Credential; import com.google.cloud.tools.jib.docker.DockerClient; import com.google.cloud.tools.jib.global.JibSystemProperties; +import com.google.cloud.tools.jib.http.Authorization; +import com.google.cloud.tools.jib.image.Image; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.function.Function; import javax.annotation.Nullable; /** @@ -40,231 +52,248 @@ */ public class StepsRunner { - /** Holds the individual steps. */ - private static class Steps { - - @Nullable private RetrieveRegistryCredentialsStep retrieveTargetRegistryCredentialsStep; - @Nullable private AuthenticatePushStep authenticatePushStep; - @Nullable private PullBaseImageStep pullBaseImageStep; - @Nullable private PullAndCacheBaseImageLayersStep pullAndCacheBaseImageLayersStep; - - @Nullable - private ImmutableList buildAndCacheApplicationLayerSteps; - - @Nullable private PushLayersStep pushBaseImageLayersStep; - @Nullable private PushLayersStep pushApplicationLayersStep; - @Nullable private BuildImageStep buildImageStep; - @Nullable private PushContainerConfigurationStep pushContainerConfigurationStep; + /** Holds the individual future results. */ + private static class StepResults { + @Nullable private Future baseImageAndAuth; + @Nullable private Future>> baseImageLayers; + @Nullable private List> applicationLayers; + @Nullable private Future builtImage; + @Nullable private Future targetRegistryCredentials; + @Nullable private Future pushAuthorization; + @Nullable private Future>> baseImageLayerPushResults; + @Nullable private Future>> applicationLayerPushResults; + @Nullable private Future containerConfigurationPushResult; + @Nullable private Future buildResult; + } - @Nullable private AsyncStep finalStep; + @VisibleForTesting + static List realizeFutures(List> futures) + throws InterruptedException, ExecutionException { + List values = new ArrayList<>(); + for (Future future : futures) { + values.add(future.get()); + } + return values; } - /** - * Starts building the steps to run. - * - * @param buildConfiguration the {@link BuildConfiguration} - * @return a new {@link StepsRunner} - */ - public static StepsRunner begin(BuildConfiguration buildConfiguration) { + @VisibleForTesting + static ListeningExecutorService getListeningExecutorService( + BuildConfiguration buildConfiguration) { ExecutorService executorService = JibSystemProperties.isSerializedExecutionEnabled() ? MoreExecutors.newDirectExecutorService() : buildConfiguration.getExecutorService(); - - return new StepsRunner(MoreExecutors.listeningDecorator(executorService), buildConfiguration); + return MoreExecutors.listeningDecorator(executorService); } - private final Steps steps = new Steps(); + private final StepResults results = new StepResults(); + private Function buildPlan; private final ListeningExecutorService listeningExecutorService; private final BuildConfiguration buildConfiguration; - /** Runnable to run all the steps. */ - private Runnable stepsRunnable = () -> {}; - - /** The total number of steps added. */ - private int stepsCount = 0; - - @Nullable private String rootProgressAllocationDescription; - @Nullable private ProgressEventDispatcher rootProgressEventDispatcher; + private final Runnable rootProgressCloser; + private final Queue childProgressDispatcherSupplier = + new ArrayDeque<>(); private StepsRunner( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration) { + ListeningExecutorService listeningExecutorService, + BuildConfiguration buildConfiguration, + String rootProgressDescription, + int rootProgressUnits) { this.listeningExecutorService = listeningExecutorService; this.buildConfiguration = buildConfiguration; + + ProgressEventDispatcher rootProgressDispatcher = + ProgressEventDispatcher.newRoot( + buildConfiguration.getEventHandlers(), rootProgressDescription, rootProgressUnits); + for (int i = 0; i < rootProgressUnits; i++) { + childProgressDispatcherSupplier.add(rootProgressDispatcher.newChildProducer()); + } + rootProgressCloser = () -> rootProgressDispatcher.close(); } - public StepsRunner retrieveTargetRegistryCredentials() { - return enqueueStep( - () -> - steps.retrieveTargetRegistryCredentialsStep = - RetrieveRegistryCredentialsStep.forTargetImage( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer())); + private void retrieveTargetRegistryCredentials() { + results.targetRegistryCredentials = + listeningExecutorService.submit( + RetrieveRegistryCredentialsStep.forTargetImage( + buildConfiguration, childProgressDispatcherSupplier.remove())); } - public StepsRunner authenticatePush() { - return enqueueStep( - () -> - steps.authenticatePushStep = + private void authenticatePush() { + results.pushAuthorization = + listeningExecutorService.submit( + () -> new AuthenticatePushStep( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer(), - Preconditions.checkNotNull(steps.retrieveTargetRegistryCredentialsStep))); + buildConfiguration, + childProgressDispatcherSupplier.remove(), + Preconditions.checkNotNull(results.targetRegistryCredentials).get()) + .call()); } - public StepsRunner pullBaseImage() { - return enqueueStep( - () -> - steps.pullBaseImageStep = - new PullBaseImageStep( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer())); + private void pullBaseImage() { + results.baseImageAndAuth = + listeningExecutorService.submit( + new PullBaseImageStep(buildConfiguration, childProgressDispatcherSupplier.remove())); } - public StepsRunner pullAndCacheBaseImageLayers() { - return enqueueStep( - () -> - steps.pullAndCacheBaseImageLayersStep = - new PullAndCacheBaseImageLayersStep( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer(), - Preconditions.checkNotNull(steps.pullBaseImageStep))); + private void pullAndCacheBaseImageLayers() { + results.baseImageLayers = + listeningExecutorService.submit( + () -> + scheduleCallables( + PullAndCacheBaseImageLayersStep.makeList( + buildConfiguration, + childProgressDispatcherSupplier.remove(), + results.baseImageAndAuth.get()))); } - public StepsRunner pushBaseImageLayers() { - return enqueueStep( - () -> - steps.pushBaseImageLayersStep = - new PushLayersStep( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer(), - Preconditions.checkNotNull(steps.authenticatePushStep), - Preconditions.checkNotNull(steps.pullAndCacheBaseImageLayersStep))); + private void pushBaseImageLayers() { + results.baseImageLayerPushResults = + listeningExecutorService.submit( + () -> + scheduleCallables( + PushLayerStep.makeList( + buildConfiguration, + childProgressDispatcherSupplier.remove(), + Preconditions.checkNotNull(results.pushAuthorization).get(), + Preconditions.checkNotNull(results.baseImageLayers).get()))); } - public StepsRunner buildAndCacheApplicationLayers() { - return enqueueStep( - () -> - steps.buildAndCacheApplicationLayerSteps = - BuildAndCacheApplicationLayerStep.makeList( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer())); + private void buildAndCacheApplicationLayers() { + results.applicationLayers = + scheduleCallables( + BuildAndCacheApplicationLayerStep.makeList( + buildConfiguration, childProgressDispatcherSupplier.remove())); } - public StepsRunner buildImage() { - return enqueueStep( - () -> - steps.buildImageStep = + private void buildImage() { + results.builtImage = + listeningExecutorService.submit( + () -> new BuildImageStep( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer(), - Preconditions.checkNotNull(steps.pullBaseImageStep), - Preconditions.checkNotNull(steps.pullAndCacheBaseImageLayersStep), - Preconditions.checkNotNull(steps.buildAndCacheApplicationLayerSteps))); + buildConfiguration, + childProgressDispatcherSupplier.remove(), + Preconditions.checkNotNull(results.baseImageAndAuth).get().getImage(), + realizeFutures(Preconditions.checkNotNull(results.baseImageLayers).get()), + realizeFutures(Preconditions.checkNotNull(results.applicationLayers))) + .call()); } - public StepsRunner pushContainerConfiguration() { - return enqueueStep( - () -> - steps.pushContainerConfigurationStep = + private void pushContainerConfiguration() { + results.containerConfigurationPushResult = + listeningExecutorService.submit( + () -> new PushContainerConfigurationStep( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer(), - Preconditions.checkNotNull(steps.authenticatePushStep), - Preconditions.checkNotNull(steps.buildImageStep))); + buildConfiguration, + childProgressDispatcherSupplier.remove(), + Preconditions.checkNotNull(results.pushAuthorization).get(), + Preconditions.checkNotNull(results.builtImage).get()) + .call()); } - public StepsRunner pushApplicationLayers() { - return enqueueStep( - () -> - steps.pushApplicationLayersStep = - new PushLayersStep( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer(), - Preconditions.checkNotNull(steps.authenticatePushStep), - AsyncSteps.immediate( - Preconditions.checkNotNull(steps.buildAndCacheApplicationLayerSteps)))); + private void pushApplicationLayers() { + results.applicationLayerPushResults = + listeningExecutorService.submit( + () -> + scheduleCallables( + PushLayerStep.makeList( + buildConfiguration, + childProgressDispatcherSupplier.remove(), + Preconditions.checkNotNull(results.pushAuthorization).get(), + Preconditions.checkNotNull(results.applicationLayers)))); } - public StepsRunner pushImage() { - rootProgressAllocationDescription = "building image to registry"; - - return enqueueStep( - () -> - steps.finalStep = - new PushImageStep( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer(), - Preconditions.checkNotNull(steps.authenticatePushStep), - Preconditions.checkNotNull(steps.pushBaseImageLayersStep), - Preconditions.checkNotNull(steps.pushApplicationLayersStep), - Preconditions.checkNotNull(steps.pushContainerConfigurationStep), - Preconditions.checkNotNull(steps.buildImageStep))); + private BuildResult pushImage() throws InterruptedException, ExecutionException, IOException { + realizeFutures(Preconditions.checkNotNull(results.baseImageLayerPushResults).get()); + realizeFutures(Preconditions.checkNotNull(results.applicationLayerPushResults).get()); + + return new PushImageStep( + listeningExecutorService, + buildConfiguration, + childProgressDispatcherSupplier.remove(), + Preconditions.checkNotNull(results.pushAuthorization).get(), + Preconditions.checkNotNull(results.containerConfigurationPushResult).get(), + Preconditions.checkNotNull(results.builtImage).get()) + .call(); } - public StepsRunner loadDocker(DockerClient dockerClient) { - rootProgressAllocationDescription = "building image to Docker daemon"; - - return enqueueStep( - () -> - steps.finalStep = - new LoadDockerStep( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer(), - dockerClient, - Preconditions.checkNotNull(steps.pullAndCacheBaseImageLayersStep), - Preconditions.checkNotNull(steps.buildAndCacheApplicationLayerSteps), - Preconditions.checkNotNull(steps.buildImageStep))); + private BuildResult loadDocker(DockerClient dockerClient) + throws InterruptedException, ExecutionException, IOException { + realizeFutures(Preconditions.checkNotNull(results.baseImageLayers).get()); + realizeFutures(Preconditions.checkNotNull(results.applicationLayers)); + + return new LoadDockerStep( + buildConfiguration, + childProgressDispatcherSupplier.remove(), + dockerClient, + Preconditions.checkNotNull(results.builtImage.get())) + .call(); } - public StepsRunner writeTarFile(Path outputPath) { - rootProgressAllocationDescription = "building image to tar file"; - - return enqueueStep( - () -> - steps.finalStep = - new WriteTarFileStep( - listeningExecutorService, - buildConfiguration, - Preconditions.checkNotNull(rootProgressEventDispatcher).newChildProducer(), - outputPath, - Preconditions.checkNotNull(steps.pullAndCacheBaseImageLayersStep), - Preconditions.checkNotNull(steps.buildAndCacheApplicationLayerSteps), - Preconditions.checkNotNull(steps.buildImageStep))); + private BuildResult writeTarFile(Path outputPath) + throws InterruptedException, ExecutionException, IOException { + realizeFutures(Preconditions.checkNotNull(results.baseImageLayers).get()); + realizeFutures(Preconditions.checkNotNull(results.applicationLayers)); + + return new WriteTarFileStep( + buildConfiguration, + childProgressDispatcherSupplier.remove(), + outputPath, + Preconditions.checkNotNull(results.builtImage).get()) + .call(); } - public BuildResult run() throws ExecutionException, InterruptedException { - Preconditions.checkNotNull(rootProgressAllocationDescription); + public BuildResult run(BuildConfiguration buildConfiguration) + throws ExecutionException, InterruptedException { + Preconditions.checkState(childProgressDispatcherSupplier.isEmpty()); - try (ProgressEventDispatcher progressEventDispatcher = - ProgressEventDispatcher.newRoot( - buildConfiguration.getEventHandlers(), rootProgressAllocationDescription, stepsCount)) { - rootProgressEventDispatcher = progressEventDispatcher; - stepsRunnable.run(); - return Preconditions.checkNotNull(steps.finalStep).getFuture().get(); + try { + return Preconditions.checkNotNull(results.buildResult).get(); + } finally { + rootProgressCloser.run(); } } - private StepsRunner enqueueStep(Runnable stepRunnable) { - Runnable previousStepsRunnable = stepsRunnable; - stepsRunnable = - () -> { - previousStepsRunnable.run(); - stepRunnable.run(); - }; - stepsCount++; + private void buildAndCache() { + pullBaseImage(); + pullAndCacheBaseImageLayers(); + buildAndCacheApplicationLayers(); + buildImage(); + } + + public StepsRunner forDockerBuild(DockerClient dockerClient) { + buildAndCache(); + loadDocker(dockerClient); + return this; + } + + public StepsRunner buildToTar(Path outputPath) { + // rootProgressDescription = "building image to Docker daemon"; + // rootProgressDescription = "building image to tar file"; + // rootProgressDescription = "building image to registry"; + + buildAndCache(); + writeTarFile(outputPath); return this; } + + public StepsRunner buildToRegistry(BuildConfiguration buildConfiguration) { + buildAndCache(); + retrieveTargetRegistryCredentials(); + authenticatePush(); + pushBaseImageLayers(); + pushContainerConfiguration(); + pushApplicationLayers(); + pushImage(); + return this; + } + + private List> scheduleCallables(ImmutableList> tasks) { + List> futures = new ArrayList<>(); + for (Callable task : tasks) { + futures.add(listeningExecutorService.submit(task)); + } + return futures; + } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java index da6e8e5028..12966dbde9 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/WriteTarFileStep.java @@ -16,18 +16,12 @@ package com.google.cloud.tools.jib.builder.steps; -import com.google.cloud.tools.jib.async.AsyncDependencies; -import com.google.cloud.tools.jib.async.AsyncStep; -import com.google.cloud.tools.jib.async.NonBlockingSteps; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.ImageTarball; import com.google.cloud.tools.jib.event.events.LogEvent; import com.google.cloud.tools.jib.filesystem.FileOperations; import com.google.cloud.tools.jib.image.Image; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; import java.io.BufferedOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -36,75 +30,42 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; -public class WriteTarFileStep implements AsyncStep, Callable { +public class WriteTarFileStep implements Callable { - private final ListeningExecutorService listeningExecutorService; private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; private final Path outputPath; - private final PullAndCacheBaseImageLayersStep pullAndCacheBaseImageLayersStep; - private final ImmutableList buildAndCacheApplicationLayerSteps; - private final BuildImageStep buildImageStep; - - private final ListenableFuture listenableFuture; + private final Image builtImage; WriteTarFileStep( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Path outputPath, - PullAndCacheBaseImageLayersStep pullAndCacheBaseImageLayersStep, - ImmutableList buildAndCacheApplicationLayerSteps, - BuildImageStep buildImageStep) { - this.listeningExecutorService = listeningExecutorService; + Image builtImage) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.outputPath = outputPath; - this.pullAndCacheBaseImageLayersStep = pullAndCacheBaseImageLayersStep; - this.buildAndCacheApplicationLayerSteps = buildAndCacheApplicationLayerSteps; - this.buildImageStep = buildImageStep; - - listenableFuture = - AsyncDependencies.using(listeningExecutorService) - .addStep(pullAndCacheBaseImageLayersStep) - .addStep(buildImageStep) - .whenAllSucceed(this); - } - - @Override - public ListenableFuture getFuture() { - return listenableFuture; + this.builtImage = builtImage; } @Override - public BuildResult call() throws ExecutionException, InterruptedException { - return AsyncDependencies.using(listeningExecutorService) - .addSteps(NonBlockingSteps.get(pullAndCacheBaseImageLayersStep)) - .addSteps(buildAndCacheApplicationLayerSteps) - .addStep(NonBlockingSteps.get(buildImageStep)) - .whenAllSucceed(this::writeTarFile) - .get(); - } - - private BuildResult writeTarFile() throws ExecutionException, IOException { + public BuildResult call() throws ExecutionException, InterruptedException, IOException { buildConfiguration .getEventHandlers() .dispatch(LogEvent.progress("Building image to tar file...")); try (ProgressEventDispatcher ignored = progressEventDispatcherFactory.create("writing to tar file", 1)) { - Image image = NonBlockingSteps.get(NonBlockingSteps.get(buildImageStep)); - // Builds the image to a tarball. Files.createDirectories(outputPath.getParent()); try (OutputStream outputStream = new BufferedOutputStream(FileOperations.newLockingOutputStream(outputPath))) { - new ImageTarball(image, buildConfiguration.getTargetImageConfiguration().getImage()) + new ImageTarball(builtImage, buildConfiguration.getTargetImageConfiguration().getImage()) .writeTo(outputStream); } - return BuildResult.fromImage(image, buildConfiguration.getTargetFormat()); + return BuildResult.fromImage(builtImage, buildConfiguration.getTargetFormat()); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java index a63a3572f3..a427db0c70 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java @@ -25,13 +25,13 @@ import com.google.cloud.tools.jib.filesystem.LockFile; import com.google.cloud.tools.jib.filesystem.TemporaryDirectory; import com.google.cloud.tools.jib.hash.CountingDigestOutputStream; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate; import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; import com.google.cloud.tools.jib.json.JsonTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.base.Preconditions; -import com.google.common.io.ByteStreams; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.IOException; @@ -101,14 +101,9 @@ private static void moveIfDoesNotExist(Path source, Path destination) throws IOE */ private static DescriptorDigest getDiffIdByDecompressingFile(Path compressedFile) throws IOException { - try (CountingDigestOutputStream diffIdCaptureOutputStream = - new CountingDigestOutputStream(ByteStreams.nullOutputStream())) { - try (InputStream fileInputStream = - new BufferedInputStream(Files.newInputStream(compressedFile)); - GZIPInputStream decompressorStream = new GZIPInputStream(fileInputStream)) { - ByteStreams.copy(decompressorStream, diffIdCaptureOutputStream); - } - return diffIdCaptureOutputStream.computeDigest().getDigest(); + try (InputStream fileInputStream = + new BufferedInputStream(new GZIPInputStream(Files.newInputStream(compressedFile)))) { + return Digests.computeDigest(fileInputStream).getDigest(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java index b6309455a1..407c333348 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java @@ -219,7 +219,6 @@ public T getManifestTemplate( layer.getBlobDescriptor().getSize(), layer.getBlobDescriptor().getDigest()); } - // Serializes into JSON. return template; } catch (InstantiationException diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java index 3da452d9c1..fb2907a923 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java @@ -137,7 +137,7 @@ public void setUp() throws DigestException { Mockito.when(mockPullBaseImageStep.getFuture()) .thenReturn( Futures.immediateFuture( - new PullBaseImageStep.BaseImageWithAuthorization(baseImage, null))); + new PullBaseImageStep.ImageAndAuthorization(baseImage, null))); Stream.of( mockBuildAndCacheApplicationLayerStepClasses, From 705dc6724822115debc1fd995621ec051d61be92 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Sat, 8 Jun 2019 00:26:46 -0400 Subject: [PATCH 02/50] wip --- .../cloud/tools/jib/api/Containerizer.java | 55 ++-- .../tools/jib/builder/steps/StepsRunner.java | 295 ++++++++++-------- 2 files changed, 196 insertions(+), 154 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java index a83c2b7284..6bc0b1ef68 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java @@ -68,7 +68,19 @@ public static Containerizer to(RegistryImage registryImage) { .setCredentialRetrievers(registryImage.getCredentialRetrievers()) .build(); - Function stepsRunnerFactory = StepsRunner::forRegistryBuild; + Function stepsRunnerFactory = + buildConfiguration -> + StepsRunner.begin(buildConfiguration) + .retrieveTargetRegistryCredentialsStep() + .authenticatePushStep() + .pullBaseImageStep() + .pullAndCacheBaseImageLayersStep() + .pushBaseImageLayersStep() + .buildAndCacheApplicationLayersStep() + .buildImageStep() + .pushContainerConfigurationStep() + .pushApplicationLayersStep() + .pushImageStep(); return new Containerizer( DESCRIPTION_FOR_DOCKER_REGISTRY, imageConfiguration, stepsRunnerFactory, true); @@ -89,12 +101,17 @@ public static Containerizer to(DockerDaemonImage dockerDaemonImage) { dockerClientBuilder.setDockerEnvironment( ImmutableMap.copyOf(dockerDaemonImage.getDockerEnvironment())); - Containerizer.Runner buildToDocker = + Function stepsRunnerFactory = buildConfiguration -> - StepsRunner.begin(buildConfiguration).buildToDocker(dockerClientBuilder.build()); + StepsRunner.begin(buildConfiguration) + .pullBaseImageStep() + .pullAndCacheBaseImageLayersStep() + .buildAndCacheApplicationLayersStep() + .buildImageStep() + .loadDockerStep(dockerClientBuilder.build()); return new Containerizer( - DESCRIPTION_FOR_DOCKER_DAEMON, imageConfiguration, buildToDocker, false); + DESCRIPTION_FOR_DOCKER_DAEMON, imageConfiguration, stepsRunnerFactory, false); } /** @@ -107,11 +124,17 @@ public static Containerizer to(TarImage tarImage) { ImageConfiguration imageConfiguration = ImageConfiguration.builder(tarImage.getImageReference()).build(); - Containerizer.Runner buildToTar = + Function stepsRunnerFactory = buildConfiguration -> - StepsRunner.begin(buildConfiguration).buildToTar(tarImage.getOutputFile()); + StepsRunner.begin(buildConfiguration) + .pullBaseImageStep() + .pullAndCacheBaseImageLayersStep() + .buildAndCacheApplicationLayersStep() + .buildImageStep() + .writeTarFileStep(tarImage.getOutputFile()); - return new Containerizer(DESCRIPTION_FOR_TARBALL, imageConfiguration, buildToTar, false); + return new Containerizer( + DESCRIPTION_FOR_TARBALL, imageConfiguration, stepsRunnerFactory, false); } private final String description; @@ -128,23 +151,15 @@ public static Containerizer to(TarImage tarImage) { private boolean offline = false; private String toolName = DEFAULT_TOOL_NAME; - public static interface Runner { - - BuildResult build(BuildConfiguration buildConfiguration) - throws InterruptedException, ExecutionException; - } - - private Runner runner; - /** Instantiate with {@link #to}. */ private Containerizer( String description, ImageConfiguration imageConfiguration, - Runner runner, + Function stepsRunnerFactory, boolean mustBeOnline) { this.description = description; this.imageConfiguration = imageConfiguration; - this.runner = runner; + this.stepsRunnerFactory = stepsRunnerFactory; this.mustBeOnline = mustBeOnline; } @@ -292,7 +307,7 @@ Path getApplicationLayersCacheDirectory() throws CacheDirectoryCreationException try { Path temporaryDirectory = Files.createTempDirectory(null); temporaryDirectory.toFile().deleteOnExit(); - this.applicationLayersCacheDirectory = temporaryDirectory; + applicationLayersCacheDirectory = temporaryDirectory; } catch (IOException ex) { throw new CacheDirectoryCreationException(ex); @@ -326,7 +341,7 @@ ImageConfiguration getImageConfiguration() { } BuildResult run(BuildConfiguration buildConfiguration) - throws InterruptedException, ExecutionException { - return runner.build(buildConfiguration); + throws ExecutionException, InterruptedException { + return stepsRunnerFactory.apply(buildConfiguration).run(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index aac0a0a58c..cf6acb91c1 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -25,20 +25,20 @@ import com.google.cloud.tools.jib.global.JibSystemProperties; import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.image.Image; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import java.nio.file.Path; -import java.util.ArrayDeque; import java.util.ArrayList; import java.util.List; -import java.util.Queue; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.function.Supplier; +import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -50,253 +50,280 @@ */ public class StepsRunner { - /** Holds the individual future results. */ + /** Holds the individual step results. */ private static class StepResults { - @Nullable private Future baseImageAndAuth; - @Nullable private Future>> baseImageLayers; - @Nullable private List> applicationLayers; - @Nullable private Future builtImage; - @Nullable private Future targetRegistryCredentials; - @Nullable private Future pushAuthorization; - @Nullable private Future>> baseImageLayerPushResults; - @Nullable private Future>> applicationLayerPushResults; - @Nullable private Future containerConfigurationPushResult; - @Nullable private Future buildResult; - } - - @VisibleForTesting - static List realizeFutures(List> futures) - throws InterruptedException, ExecutionException { - List values = new ArrayList<>(); - for (Future future : futures) { - values.add(future.get()); + private static Future failedFuture() { + return Futures.immediateFailedFuture( + new IllegalStateException("invalid usage; required step not configured")); } - return values; + + private Future baseImageAndAuth = failedFuture(); + private Future>> baseImageLayers = failedFuture(); + private List> applicationLayers = new ArrayList<>(); + private Future builtImage = failedFuture(); + private Future targetRegistryCredentials = failedFuture(); + private Future pushAuthorization = failedFuture(); + private Future>> baseImageLayerPushResults = failedFuture(); + private Future>> applicationLayerPushResults = failedFuture(); + private Future containerConfigurationPushResult = failedFuture(); + private Future buildResult = failedFuture(); } - @VisibleForTesting - static ListeningExecutorService getListeningExecutorService( - BuildConfiguration buildConfiguration) { + /** + * Starts building the steps to run. + * + * @param buildConfiguration the {@link BuildConfiguration} + * @return a new {@link StepsRunner} + */ + public static StepsRunner begin(BuildConfiguration buildConfiguration) { ExecutorService executorService = JibSystemProperties.isSerializedExecutionEnabled() ? MoreExecutors.newDirectExecutorService() : buildConfiguration.getExecutorService(); - return MoreExecutors.listeningDecorator(executorService); + + return new StepsRunner(MoreExecutors.listeningDecorator(executorService), buildConfiguration); } private final StepResults results = new StepResults(); - private Callable buildPlan; - private final ListeningExecutorService listeningExecutorService; + // TODO: use plain ExecutorService; requires refactoring PushImageStep. + private final ListeningExecutorService executorService; private final BuildConfiguration buildConfiguration; - private final Runnable rootProgressCloser; - private final Queue childProgressDispatcherSupplier = - new ArrayDeque<>(); + private final List stepsToRun = new ArrayList<>(); + + @Nullable private String rootProgressDescription; + private Supplier childProgressDispatcherFactorySupplier = + () -> { + throw new IllegalStateException("root progress dispatcher uninstantiated"); + }; private StepsRunner( - ListeningExecutorService listeningExecutorService, - BuildConfiguration buildConfiguration, - String rootProgressDescription, - int rootProgressUnits) { - this.listeningExecutorService = listeningExecutorService; + ListeningExecutorService executorService, BuildConfiguration buildConfiguration) { + this.executorService = executorService; this.buildConfiguration = buildConfiguration; - - ProgressEventDispatcher rootProgressDispatcher = - ProgressEventDispatcher.newRoot( - buildConfiguration.getEventHandlers(), rootProgressDescription, rootProgressUnits); - for (int i = 0; i < rootProgressUnits; i++) { - childProgressDispatcherSupplier.add(rootProgressDispatcher.newChildProducer()); - } - rootProgressCloser = () -> rootProgressDispatcher.close(); } private void retrieveTargetRegistryCredentials() { results.targetRegistryCredentials = - listeningExecutorService.submit( + executorService.submit( RetrieveRegistryCredentialsStep.forTargetImage( - buildConfiguration, childProgressDispatcherSupplier.remove())); + buildConfiguration, childProgressDispatcherFactorySupplier.get())); } private void authenticatePush() { results.pushAuthorization = - listeningExecutorService.submit( + executorService.submit( () -> new AuthenticatePushStep( buildConfiguration, - childProgressDispatcherSupplier.remove(), + childProgressDispatcherFactorySupplier.get(), Preconditions.checkNotNull(results.targetRegistryCredentials).get()) .call()); } private void pullBaseImage() { results.baseImageAndAuth = - listeningExecutorService.submit( - new PullBaseImageStep(buildConfiguration, childProgressDispatcherSupplier.remove())); + executorService.submit( + new PullBaseImageStep( + buildConfiguration, childProgressDispatcherFactorySupplier.get())); } private void pullAndCacheBaseImageLayers() { results.baseImageLayers = - listeningExecutorService.submit( + executorService.submit( () -> scheduleCallables( PullAndCacheBaseImageLayersStep.makeList( buildConfiguration, - childProgressDispatcherSupplier.remove(), + childProgressDispatcherFactorySupplier.get(), results.baseImageAndAuth.get()))); } private void pushBaseImageLayers() { results.baseImageLayerPushResults = - listeningExecutorService.submit( + executorService.submit( () -> scheduleCallables( PushLayerStep.makeList( buildConfiguration, - childProgressDispatcherSupplier.remove(), - Preconditions.checkNotNull(results.pushAuthorization).get(), - Preconditions.checkNotNull(results.baseImageLayers).get()))); + childProgressDispatcherFactorySupplier.get(), + results.pushAuthorization.get(), + results.baseImageLayers.get()))); } private void buildAndCacheApplicationLayers() { results.applicationLayers = scheduleCallables( BuildAndCacheApplicationLayerStep.makeList( - buildConfiguration, childProgressDispatcherSupplier.remove())); + buildConfiguration, childProgressDispatcherFactorySupplier.get())); } private void buildImage() { results.builtImage = - listeningExecutorService.submit( + executorService.submit( () -> new BuildImageStep( buildConfiguration, - childProgressDispatcherSupplier.remove(), + childProgressDispatcherFactorySupplier.get(), Preconditions.checkNotNull(results.baseImageAndAuth).get().getImage(), - realizeFutures(Preconditions.checkNotNull(results.baseImageLayers).get()), - realizeFutures(Preconditions.checkNotNull(results.applicationLayers))) + realizeFutures(results.baseImageLayers.get()), + realizeFutures(results.applicationLayers)) .call()); } private void pushContainerConfiguration() { results.containerConfigurationPushResult = - listeningExecutorService.submit( + executorService.submit( () -> new PushContainerConfigurationStep( buildConfiguration, - childProgressDispatcherSupplier.remove(), - Preconditions.checkNotNull(results.pushAuthorization).get(), - Preconditions.checkNotNull(results.builtImage).get()) + childProgressDispatcherFactorySupplier.get(), + results.pushAuthorization.get(), + results.builtImage.get()) .call()); } private void pushApplicationLayers() { results.applicationLayerPushResults = - listeningExecutorService.submit( + executorService.submit( () -> scheduleCallables( PushLayerStep.makeList( buildConfiguration, - childProgressDispatcherSupplier.remove(), - Preconditions.checkNotNull(results.pushAuthorization).get(), - Preconditions.checkNotNull(results.applicationLayers)))); + childProgressDispatcherFactorySupplier.get(), + results.pushAuthorization.get(), + results.applicationLayers))); } private void pushImage() { results.buildResult = - listeningExecutorService.submit( + executorService.submit( () -> { - realizeFutures(Preconditions.checkNotNull(results.baseImageLayerPushResults).get()); - realizeFutures(Preconditions.checkNotNull(results.applicationLayerPushResults).get()); + realizeFutures(results.baseImageLayerPushResults.get()); + realizeFutures(results.applicationLayerPushResults.get()); return new PushImageStep( - listeningExecutorService, + executorService, buildConfiguration, - childProgressDispatcherSupplier.remove(), - Preconditions.checkNotNull(results.pushAuthorization).get(), - Preconditions.checkNotNull(results.containerConfigurationPushResult).get(), - Preconditions.checkNotNull(results.builtImage).get()) + childProgressDispatcherFactorySupplier.get(), + results.pushAuthorization.get(), + results.containerConfigurationPushResult.get(), + results.builtImage.get()) .call(); }); } private void loadDocker(DockerClient dockerClient) { results.buildResult = - listeningExecutorService.submit( - () -> { - realizeFutures(Preconditions.checkNotNull(results.baseImageLayers).get()); - realizeFutures(Preconditions.checkNotNull(results.applicationLayers)); - - return new LoadDockerStep( - buildConfiguration, - childProgressDispatcherSupplier.remove(), - dockerClient, - Preconditions.checkNotNull(results.builtImage.get())) - .call(); - }); + executorService.submit( + () -> + new LoadDockerStep( + buildConfiguration, + childProgressDispatcherFactorySupplier.get(), + dockerClient, + results.builtImage.get()) + .call()); } private void writeTarFile(Path outputPath) { results.buildResult = - listeningExecutorService.submit( - () -> { - realizeFutures(Preconditions.checkNotNull(results.baseImageLayers).get()); - realizeFutures(Preconditions.checkNotNull(results.applicationLayers)); - - return new WriteTarFileStep( - buildConfiguration, - childProgressDispatcherSupplier.remove(), - outputPath, - Preconditions.checkNotNull(results.builtImage).get()) - .call(); - }); + executorService.submit( + () -> + new WriteTarFileStep( + buildConfiguration, + childProgressDispatcherFactorySupplier.get(), + outputPath, + results.builtImage.get()) + .call()); } - public BuildResult run(BuildConfiguration buildConfiguration) - throws ExecutionException, InterruptedException { - Preconditions.checkState(childProgressDispatcherSupplier.isEmpty()); + public BuildResult run() throws ExecutionException, InterruptedException { + Preconditions.checkNotNull(rootProgressDescription); + + try (ProgressEventDispatcher progressEventDispatcher = + ProgressEventDispatcher.newRoot( + buildConfiguration.getEventHandlers(), rootProgressDescription, stepsToRun.size())) { + childProgressDispatcherFactorySupplier = progressEventDispatcher::newChildProducer; + stepsToRun.forEach(Runnable::run); + return results.buildResult.get(); + } + } - try { - return Preconditions.checkNotNull(results.buildResult).get(); - } finally { - rootProgressCloser.run(); + private static List realizeFutures(List> futures) + throws InterruptedException, ExecutionException { + List values = new ArrayList<>(); + for (Future future : futures) { + values.add(future.get()); } + return values; } - private void buildAndCache() { - pullBaseImage(); - pullAndCacheBaseImageLayers(); - buildAndCacheApplicationLayers(); - buildImage(); + private List> scheduleCallables(ImmutableList> callables) { + return callables.stream().map(executorService::submit).collect(Collectors.toList()); } - public StepsRunner buildToTar(Path outputPath) { - // rootProgressDescription = "building image to Docker daemon"; - // rootProgressDescription = "building image to tar file"; - // rootProgressDescription = "building image to registry"; + public StepsRunner pullBaseImageStep() { + stepsToRun.add(this::pullBaseImage); + return this; + } - buildAndCache(); - // writeTarFile(outputPath); + public StepsRunner pullAndCacheBaseImageLayersStep() { + stepsToRun.add(this::pullAndCacheBaseImageLayers); return this; } - public StepsRunner buildToRegistry(BuildConfiguration buildConfiguration) { - buildAndCache(); - retrieveTargetRegistryCredentials(); - authenticatePush(); - pushBaseImageLayers(); - pushContainerConfiguration(); - pushApplicationLayers(); - // pushImage(); + public StepsRunner buildAndCacheApplicationLayersStep() { + stepsToRun.add(this::buildAndCacheApplicationLayers); return this; } - private List> scheduleCallables(ImmutableList> tasks) { - List> futures = new ArrayList<>(); - for (Callable task : tasks) { - futures.add(listeningExecutorService.submit(task)); - } - return futures; + public StepsRunner buildImageStep() { + stepsToRun.add(this::buildImage); + return this; + } + + public StepsRunner loadDockerStep(DockerClient dockerClient) { + rootProgressDescription = "building image to Docker daemon"; + stepsToRun.add(() -> loadDocker(dockerClient)); + return this; + } + + public StepsRunner writeTarFileStep(Path outputPath) { + rootProgressDescription = "building image to tar file"; + stepsToRun.add(() -> writeTarFile(outputPath)); + return this; + } + + public StepsRunner retrieveTargetRegistryCredentialsStep() { + stepsToRun.add(this::retrieveTargetRegistryCredentials); + return this; + } + + public StepsRunner authenticatePushStep() { + stepsToRun.add(this::authenticatePush); + return this; + } + + public StepsRunner pushBaseImageLayersStep() { + stepsToRun.add(this::pushBaseImageLayers); + return this; + } + + public StepsRunner pushApplicationLayersStep() { + stepsToRun.add(this::pushApplicationLayers); + return this; + } + + public StepsRunner pushContainerConfigurationStep() { + stepsToRun.add(this::pushContainerConfiguration); + return this; + } + + public StepsRunner pushImageStep() { + rootProgressDescription = "building image to registry"; + stepsToRun.add(this::pushImage); + return this; } } From 449744d1be413c5e30fc3909ea6b21b77688589c Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Sat, 8 Jun 2019 00:38:35 -0400 Subject: [PATCH 03/50] wip --- .../cloud/tools/jib/api/Containerizer.java | 27 ++-------- .../tools/jib/builder/steps/StepsRunner.java | 50 ++++--------------- 2 files changed, 13 insertions(+), 64 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java index 6bc0b1ef68..c9526c08b0 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java @@ -69,18 +69,7 @@ public static Containerizer to(RegistryImage registryImage) { .build(); Function stepsRunnerFactory = - buildConfiguration -> - StepsRunner.begin(buildConfiguration) - .retrieveTargetRegistryCredentialsStep() - .authenticatePushStep() - .pullBaseImageStep() - .pullAndCacheBaseImageLayersStep() - .pushBaseImageLayersStep() - .buildAndCacheApplicationLayersStep() - .buildImageStep() - .pushContainerConfigurationStep() - .pushApplicationLayersStep() - .pushImageStep(); + buildConfiguration -> StepsRunner.begin(buildConfiguration).registryPushSteps(); return new Containerizer( DESCRIPTION_FOR_DOCKER_REGISTRY, imageConfiguration, stepsRunnerFactory, true); @@ -103,12 +92,7 @@ public static Containerizer to(DockerDaemonImage dockerDaemonImage) { Function stepsRunnerFactory = buildConfiguration -> - StepsRunner.begin(buildConfiguration) - .pullBaseImageStep() - .pullAndCacheBaseImageLayersStep() - .buildAndCacheApplicationLayersStep() - .buildImageStep() - .loadDockerStep(dockerClientBuilder.build()); + StepsRunner.begin(buildConfiguration).dockerLoadSteps(dockerClientBuilder.build()); return new Containerizer( DESCRIPTION_FOR_DOCKER_DAEMON, imageConfiguration, stepsRunnerFactory, false); @@ -126,12 +110,7 @@ public static Containerizer to(TarImage tarImage) { Function stepsRunnerFactory = buildConfiguration -> - StepsRunner.begin(buildConfiguration) - .pullBaseImageStep() - .pullAndCacheBaseImageLayersStep() - .buildAndCacheApplicationLayersStep() - .buildImageStep() - .writeTarFileStep(tarImage.getOutputFile()); + StepsRunner.begin(buildConfiguration).tarBuildSteps(tarImage.getOutputFile()); return new Containerizer( DESCRIPTION_FOR_TARBALL, imageConfiguration, stepsRunnerFactory, false); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index cf6acb91c1..6293784860 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -119,7 +119,7 @@ private void authenticatePush() { new AuthenticatePushStep( buildConfiguration, childProgressDispatcherFactorySupplier.get(), - Preconditions.checkNotNull(results.targetRegistryCredentials).get()) + results.targetRegistryCredentials.get()) .call()); } @@ -167,7 +167,7 @@ private void buildImage() { new BuildImageStep( buildConfiguration, childProgressDispatcherFactorySupplier.get(), - Preconditions.checkNotNull(results.baseImageAndAuth).get().getImage(), + results.baseImageAndAuth.get().getImage(), realizeFutures(results.baseImageLayers.get()), realizeFutures(results.applicationLayers)) .call()); @@ -264,65 +264,35 @@ private List> scheduleCallables(ImmutableList loadDocker(dockerClient)); return this; } - public StepsRunner writeTarFileStep(Path outputPath) { + public StepsRunner tarBuildSteps(Path outputPath) { rootProgressDescription = "building image to tar file"; + enqueueBuildAndCache(); stepsToRun.add(() -> writeTarFile(outputPath)); return this; } - public StepsRunner retrieveTargetRegistryCredentialsStep() { + public StepsRunner registryPushSteps() { + rootProgressDescription = "building image to registry"; + enqueueBuildAndCache(); stepsToRun.add(this::retrieveTargetRegistryCredentials); - return this; - } - - public StepsRunner authenticatePushStep() { stepsToRun.add(this::authenticatePush); - return this; - } - - public StepsRunner pushBaseImageLayersStep() { stepsToRun.add(this::pushBaseImageLayers); - return this; - } - - public StepsRunner pushApplicationLayersStep() { stepsToRun.add(this::pushApplicationLayers); - return this; - } - - public StepsRunner pushContainerConfigurationStep() { stepsToRun.add(this::pushContainerConfiguration); - return this; - } - - public StepsRunner pushImageStep() { - rootProgressDescription = "building image to registry"; stepsToRun.add(this::pushImage); return this; } From 71567bed9c69708b7471d38ca48d3f4f435f5a51 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Sat, 8 Jun 2019 01:01:37 -0400 Subject: [PATCH 04/50] wip --- .../tools/jib/async/AsyncDependencies.java | 84 ------------------- .../cloud/tools/jib/async/AsyncStep.java | 42 ---------- .../cloud/tools/jib/async/AsyncSteps.java | 31 ------- .../tools/jib/async/NonBlockingSteps.java | 42 ---------- .../builder/steps/AuthenticatePushStep.java | 8 +- .../BuildAndCacheApplicationLayerStep.java | 2 +- .../jib/builder/steps/BuildImageStep.java | 4 +- .../jib/builder/steps/CachedLayerAndName.java | 18 +++- .../jib/builder/steps/LoadDockerStep.java | 3 +- .../jib/builder/steps/PullBaseImageStep.java | 3 +- .../jib/api/JibContainerBuilderTest.java | 7 +- ...BuildAndCacheApplicationLayerStepTest.java | 15 ++-- .../RetrieveRegistryCredentialsStepTest.java | 7 -- 13 files changed, 31 insertions(+), 235 deletions(-) delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncDependencies.java delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncStep.java delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncSteps.java delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/async/NonBlockingSteps.java diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncDependencies.java b/jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncDependencies.java deleted file mode 100644 index 7f43957e94..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncDependencies.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.async; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; - -/** - * Builds a list of dependency {@link ListenableFuture}s to wait on before calling a {@link - * Callable}. - */ -public class AsyncDependencies { - - /** - * Initialize with a {@link ListeningExecutorService}. - * - * @param listeningExecutorService the {@link ListeningExecutorService} - * @return a new {@link AsyncDependencies} - */ - public static AsyncDependencies using(ListeningExecutorService listeningExecutorService) { - return new AsyncDependencies(listeningExecutorService); - } - - private final ListeningExecutorService listeningExecutorService; - - /** Stores the list of {@link ListenableFuture}s to wait on. */ - private final List> futures = new ArrayList<>(); - - private AsyncDependencies(ListeningExecutorService listeningExecutorService) { - this.listeningExecutorService = listeningExecutorService; - } - - /** - * Adds the future of an {@link AsyncStep}. - * - * @param asyncStep the {@link AsyncStep} - * @return this - */ - public AsyncDependencies addStep(AsyncStep asyncStep) { - futures.add(asyncStep.getFuture()); - return this; - } - - /** - * Adds the futures of a list of {@link AsyncStep}s. - * - * @param asyncSteps the {@link AsyncStep}s - * @return this - */ - public AsyncDependencies addSteps(List> asyncSteps) { - asyncSteps.forEach(this::addStep); - return this; - } - - /** - * Creates the {@link ListenableFuture} which will return the result of calling {@code combiner} - * when all the added futures succeed. - * - * @param combiner the {@link Callable} - * @param the return type of {@code combiner} - * @return a {@link ListenableFuture} to handle completion of the call to {@code combiner} - */ - public ListenableFuture whenAllSucceed(Callable combiner) { - return Futures.whenAllSucceed(futures).call(combiner, listeningExecutorService); - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncStep.java deleted file mode 100644 index bab5363449..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncStep.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.async; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import java.util.concurrent.Callable; - -/** - * Holds the future for an asynchronously-running step. Implementations should: - * - *
    - *
  1. Be immutable - *
  2. Construct with the dependent {@link AsyncStep}s and submit a {@link Callable} to the {@link - * ListeningExecutorService} to run after all its dependent {@link AsyncStep}s (for example, - * by using {@link Futures#whenAllSucceed}) - *
  3. Have {@link #getFuture} return the submitted future - *
- * - * @param the object type passed on by this step - */ -public interface AsyncStep { - - /** @return the submitted future */ - // TODO: Consider changing this to be orchestrated by an AsyncStepsBuilder. - ListenableFuture getFuture(); -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncSteps.java b/jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncSteps.java deleted file mode 100644 index 0bb3771a0e..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/async/AsyncSteps.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.async; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; - -/** Static methods for {@link AsyncStep}. */ -public class AsyncSteps { - - public static AsyncStep immediate(T returnValue) { - ListenableFuture future = Futures.immediateFuture(returnValue); - return () -> future; - } - - private AsyncSteps() {} -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/async/NonBlockingSteps.java b/jib-core/src/main/java/com/google/cloud/tools/jib/async/NonBlockingSteps.java deleted file mode 100644 index d65b3f5c9a..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/async/NonBlockingSteps.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.async; - -import com.google.common.util.concurrent.Futures; -import java.util.concurrent.ExecutionException; - -/** - * Static utility for checking at runtime that the caller attempts to get a result only from a - * completed {@link AsyncStep} by otherwise throwing a runtime exception. - */ -public class NonBlockingSteps { - - /** - * Gets the completed computation result of {@code asyncStep}. - * - * @param the type of the computation result of {@code asyncStep} - * @param asyncStep completed {@link AsyncStep} - * @return the completed computation result - * @throws ExecutionException if the {@code Future} failed with an exception - * @throws IllegalStateException if {@code asyncStep} has not been completed - */ - public static T get(AsyncStep asyncStep) throws ExecutionException { - return Futures.getDone(asyncStep.getFuture()); - } - - private NonBlockingSteps() {} -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java index 4ac0ec73f0..d5a3ac351e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java @@ -26,7 +26,6 @@ import com.google.cloud.tools.jib.registry.RegistryAuthenticator; import java.io.IOException; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; import javax.annotation.Nullable; /** @@ -37,7 +36,7 @@ */ class AuthenticatePushStep implements Callable { - private static final String DESCRIPTION = "Authenticating with push to %s"; + private static final String DESCRIPTION = "Authenticating push to %s"; private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; @@ -49,13 +48,12 @@ class AuthenticatePushStep implements Callable { Credential targetRegistryCredentials) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.registryCredentials = targetRegistryCredentials; + registryCredentials = targetRegistryCredentials; } @Override @Nullable - public Authorization call() - throws ExecutionException, IOException, RegistryException, InterruptedException { + public Authorization call() throws IOException, RegistryException { String registry = buildConfiguration.getTargetImageConfiguration().getImageRegistry(); try (ProgressEventDispatcher ignored = progressEventDispatcherFactory.create("authenticating push to " + registry, 1); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java index 6071ba20a9..4a4de43f40 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java @@ -47,7 +47,7 @@ static ImmutableList makeList( try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create( - "preparing to application layer builders", layerCount); + "preparing application layer builders", layerCount); TimerEventDispatcher ignored = new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { ImmutableList.Builder buildAndCacheApplicationLayerSteps = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java index ab2502b92c..bb3783eb75 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java @@ -30,7 +30,6 @@ import java.time.Instant; import java.util.List; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; import javax.annotation.Nullable; /** Builds a model {@link Image}. */ @@ -58,8 +57,7 @@ class BuildImageStep implements Callable { } @Override - public Image call() - throws ExecutionException, LayerPropertyNotFoundException, InterruptedException { + public Image call() throws LayerPropertyNotFoundException { try (ProgressEventDispatcher ignored = progressEventDispatcherFactory.create("building image format", 1); TimerEventDispatcher ignored2 = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java index 3e5ec1c9aa..85e80cdd82 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java @@ -1,9 +1,25 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.cache.CachedLayer; import javax.annotation.Nullable; -/** Simple structure to hold the result pair of {#link CachedLayer} and its name. */ +/** Simple structure to hold the result pair of {#link CachedLayer} and its string name. */ class CachedLayerAndName { private CachedLayer cachedLayer; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java index 79688a2618..9663669898 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/LoadDockerStep.java @@ -26,7 +26,6 @@ import com.google.cloud.tools.jib.image.Image; import java.io.IOException; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; /** Adds image layers to a tarball and loads into Docker daemon. */ class LoadDockerStep implements Callable { @@ -49,7 +48,7 @@ class LoadDockerStep implements Callable { } @Override - public BuildResult call() throws ExecutionException, InterruptedException, IOException { + public BuildResult call() throws InterruptedException, IOException { EventHandlers eventHandlers = buildConfiguration.getEventHandlers(); eventHandlers.dispatch(LogEvent.progress("Loading to Docker daemon...")); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index 3b657fcaa5..d0de2e7cca 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -53,7 +53,6 @@ import java.io.IOException; import java.util.Optional; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; import javax.annotation.Nullable; /** Pulls the base image manifest. */ @@ -104,7 +103,7 @@ Authorization getAuthorization() { @Override public ImageAndAuthorization call() throws IOException, RegistryException, LayerPropertyNotFoundException, - LayerCountMismatchException, ExecutionException, BadContainerConfigurationFormatException, + LayerCountMismatchException, BadContainerConfigurationFormatException, CacheCorruptedException, CredentialRetrievalException { EventHandlers eventHandlers = buildConfiguration.getEventHandlers(); // Skip this step if this is a scratch image diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/api/JibContainerBuilderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/api/JibContainerBuilderTest.java index b1b7359774..eb05866a7f 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/api/JibContainerBuilderTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/api/JibContainerBuilderTest.java @@ -17,7 +17,6 @@ package com.google.cloud.tools.jib.api; import com.google.cloud.tools.jib.builder.steps.BuildResult; -import com.google.cloud.tools.jib.builder.steps.StepsRunner; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.configuration.ContainerConfiguration; import com.google.cloud.tools.jib.configuration.ImageConfiguration; @@ -263,14 +262,12 @@ private Containerizer createMockContainerizer() ImageReference targetImage = ImageReference.parse("target-image"); Containerizer mockContainerizer = Mockito.mock(Containerizer.class); - StepsRunner stepsRunner = Mockito.mock(StepsRunner.class); BuildResult mockBuildResult = Mockito.mock(BuildResult.class); Mockito.when(mockContainerizer.getImageConfiguration()) .thenReturn(ImageConfiguration.builder(targetImage).build()); - Mockito.when(mockContainerizer.createStepsRunner(Mockito.any(BuildConfiguration.class))) - .thenReturn(stepsRunner); - Mockito.when(stepsRunner.run()).thenReturn(mockBuildResult); + Mockito.when(mockContainerizer.run(Mockito.any(BuildConfiguration.class))) + .thenReturn(mockBuildResult); Mockito.when(mockBuildResult.getImageDigest()) .thenReturn( DescriptorDigest.fromHash( diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java index f3d56e4725..a8c7a76068 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.api.AbsoluteUnixPath; import com.google.cloud.tools.jib.api.LayerConfiguration; import com.google.cloud.tools.jib.api.LayerEntry; -import com.google.cloud.tools.jib.async.NonBlockingSteps; import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; @@ -32,13 +31,11 @@ import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; import com.google.common.collect.ImmutableList; import com.google.common.io.Resources; -import com.google.common.util.concurrent.MoreExecutors; import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.concurrent.ExecutionException; import java.util.stream.Stream; import org.junit.Assert; import org.junit.Before; @@ -126,18 +123,18 @@ public void setUp() throws IOException, URISyntaxException { Mockito.when(mockBuildConfiguration.getApplicationLayersCache()).thenReturn(cache); } - private ImageLayers buildFakeLayersToCache() throws ExecutionException { + private ImageLayers buildFakeLayersToCache() + throws LayerPropertyNotFoundException, IOException, CacheCorruptedException { ImageLayers.Builder applicationLayersBuilder = ImageLayers.builder(); ImmutableList buildAndCacheApplicationLayerSteps = BuildAndCacheApplicationLayerStep.makeList( - MoreExecutors.newDirectExecutorService(), mockBuildConfiguration, ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer()); for (BuildAndCacheApplicationLayerStep buildAndCacheApplicationLayerStep : buildAndCacheApplicationLayerSteps) { - applicationLayersBuilder.add(NonBlockingSteps.get(buildAndCacheApplicationLayerStep)); + applicationLayersBuilder.add(buildAndCacheApplicationLayerStep.call().getCachedLayer()); } return applicationLayersBuilder.build(); @@ -145,8 +142,7 @@ private ImageLayers buildFakeLayersToCache() throws ExecutionException { @Test public void testRun() - throws LayerPropertyNotFoundException, IOException, ExecutionException, - CacheCorruptedException { + throws LayerPropertyNotFoundException, IOException, CacheCorruptedException { ImmutableList fakeLayerConfigurations = ImmutableList.of( fakeDependenciesLayerConfiguration, @@ -207,8 +203,7 @@ public void testRun() } @Test - public void testRun_emptyLayersIgnored() - throws IOException, ExecutionException, CacheCorruptedException { + public void testRun_emptyLayersIgnored() throws IOException, CacheCorruptedException { ImmutableList fakeLayerConfigurations = ImmutableList.of( fakeDependenciesLayerConfiguration, diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStepTest.java index ad4ce0a2b4..2121f33c81 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStepTest.java @@ -26,7 +26,6 @@ import com.google.cloud.tools.jib.event.EventHandlers; import com.google.cloud.tools.jib.event.events.ProgressEvent; import com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException; -import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import java.io.IOException; import java.nio.file.Paths; @@ -46,7 +45,6 @@ public class RetrieveRegistryCredentialsStepTest { @Mock private EventHandlers mockEventHandlers; - @Mock private ListeningExecutorService mockListeningExecutorService; @Test public void testCall_retrieved() throws CredentialRetrievalException, IOException { @@ -62,14 +60,12 @@ public void testCall_retrieved() throws CredentialRetrievalException, IOExceptio Assert.assertEquals( Credential.from("baseusername", "basepassword"), RetrieveRegistryCredentialsStep.forBaseImage( - mockListeningExecutorService, buildConfiguration, ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer()) .call()); Assert.assertEquals( Credential.from("targetusername", "targetpassword"), RetrieveRegistryCredentialsStep.forTargetImage( - mockListeningExecutorService, buildConfiguration, ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer()) .call()); @@ -82,7 +78,6 @@ public void testCall_none() throws CredentialRetrievalException, IOException { Arrays.asList(Optional::empty, Optional::empty), Collections.emptyList()); Assert.assertNull( RetrieveRegistryCredentialsStep.forBaseImage( - mockListeningExecutorService, buildConfiguration, ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer()) .call()); @@ -94,7 +89,6 @@ public void testCall_none() throws CredentialRetrievalException, IOException { Assert.assertNull( RetrieveRegistryCredentialsStep.forTargetImage( - mockListeningExecutorService, buildConfiguration, ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer()) .call()); @@ -116,7 +110,6 @@ public void testCall_exception() throws IOException { Collections.emptyList()); try { RetrieveRegistryCredentialsStep.forBaseImage( - mockListeningExecutorService, buildConfiguration, ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer()) .call(); From 7d436ebf355613f8818c352878b74a90c079f831 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Sun, 9 Jun 2019 19:52:59 -0400 Subject: [PATCH 05/50] wip --- .../jib/builder/steps/BuildImageStepTest.java | 193 +++++++----------- 1 file changed, 70 insertions(+), 123 deletions(-) diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java index fb2907a923..d35d4b7c30 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java @@ -32,14 +32,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.MoreExecutors; import java.security.DigestException; import java.time.Duration; import java.time.Instant; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.stream.Stream; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -52,17 +50,15 @@ @RunWith(MockitoJUnitRunner.class) public class BuildImageStepTest { - @Mock private EventHandlers mockEventHandlers; + @Mock private ProgressEventDispatcher.Factory mockProgressEventDispatcherFactory; @Mock private BuildConfiguration mockBuildConfiguration; @Mock private ContainerConfiguration mockContainerConfiguration; - @Mock private PullBaseImageStep mockPullBaseImageStep; - @Mock private PullAndCacheBaseImageLayersStep mockPullAndCacheBaseImageLayersStep; - @Mock private PullAndCacheBaseImageLayerStep mockPullAndCacheBaseImageLayerStep; - @Mock private BuildAndCacheApplicationLayerStep mockBuildAndCacheApplicationLayerStepDependencies; - @Mock private BuildAndCacheApplicationLayerStep mockBuildAndCacheApplicationLayerStepResources; - @Mock private BuildAndCacheApplicationLayerStep mockBuildAndCacheApplicationLayerStepClasses; - @Mock private BuildAndCacheApplicationLayerStep mockBuildAndCacheApplicationLayerStepExtraFiles; @Mock private CachedLayer mockCachedLayer; + + private Image baseImage; + private List baseImageLayers; + private List applicationLayers; + private DescriptorDigest testDescriptorDigest; private HistoryEntry nonEmptyLayerHistory; private HistoryEntry emptyLayerHistory; @@ -73,7 +69,7 @@ public void setUp() throws DigestException { DescriptorDigest.fromHash( "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - Mockito.when(mockBuildConfiguration.getEventHandlers()).thenReturn(mockEventHandlers); + Mockito.when(mockBuildConfiguration.getEventHandlers()).thenReturn(EventHandlers.NONE); Mockito.when(mockBuildConfiguration.getContainerConfiguration()) .thenReturn(mockContainerConfiguration); Mockito.when(mockBuildConfiguration.getToolName()).thenReturn("jib"); @@ -100,7 +96,7 @@ public void setUp() throws DigestException { .setEmptyLayer(true) .build(); - Image baseImage = + baseImage = Image.builder(V22ManifestTemplate.class) .setArchitecture("wasm") .setOs("js") @@ -125,53 +121,29 @@ public void setUp() throws DigestException { .addHistory(emptyLayerHistory) .addHistory(emptyLayerHistory) .build(); - Mockito.when(mockPullAndCacheBaseImageLayerStep.getFuture()) - .thenReturn(Futures.immediateFuture(mockCachedLayer)); - Mockito.when(mockPullAndCacheBaseImageLayersStep.getFuture()) - .thenReturn( - Futures.immediateFuture( - ImmutableList.of( - mockPullAndCacheBaseImageLayerStep, - mockPullAndCacheBaseImageLayerStep, - mockPullAndCacheBaseImageLayerStep))); - Mockito.when(mockPullBaseImageStep.getFuture()) - .thenReturn( - Futures.immediateFuture( - new PullBaseImageStep.ImageAndAuthorization(baseImage, null))); - - Stream.of( - mockBuildAndCacheApplicationLayerStepClasses, - mockBuildAndCacheApplicationLayerStepDependencies, - mockBuildAndCacheApplicationLayerStepExtraFiles, - mockBuildAndCacheApplicationLayerStepResources) - .forEach( - layerStep -> - Mockito.when(layerStep.getFuture()) - .thenReturn(Futures.immediateFuture(mockCachedLayer))); - - Mockito.when(mockBuildAndCacheApplicationLayerStepClasses.getLayerType()).thenReturn("classes"); - Mockito.when(mockBuildAndCacheApplicationLayerStepDependencies.getLayerType()) - .thenReturn("dependencies"); - Mockito.when(mockBuildAndCacheApplicationLayerStepExtraFiles.getLayerType()) - .thenReturn("extra files"); - Mockito.when(mockBuildAndCacheApplicationLayerStepResources.getLayerType()) - .thenReturn("resources"); + baseImageLayers = + Arrays.asList( + new CachedLayerAndName(mockCachedLayer, null), + new CachedLayerAndName(mockCachedLayer, null), + new CachedLayerAndName(mockCachedLayer, null)); + applicationLayers = + Arrays.asList( + new CachedLayerAndName(mockCachedLayer, "dependencies"), + new CachedLayerAndName(mockCachedLayer, "resources"), + new CachedLayerAndName(mockCachedLayer, "classes"), + new CachedLayerAndName(mockCachedLayer, "extra files")); } @Test public void test_validateAsyncDependencies() throws ExecutionException, InterruptedException { - BuildImageStep buildImageStep = + Image image = new BuildImageStep( - MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()), - mockBuildConfiguration, - ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer(), - mockPullBaseImageStep, - mockPullAndCacheBaseImageLayersStep, - ImmutableList.of( - mockBuildAndCacheApplicationLayerStepDependencies, - mockBuildAndCacheApplicationLayerStepResources, - mockBuildAndCacheApplicationLayerStepClasses)); - Image image = buildImageStep.getFuture().get().getFuture().get(); + mockBuildConfiguration, + mockProgressEventDispatcherFactory, + baseImage, + baseImageLayers, + applicationLayers) + .call(); Assert.assertEquals( testDescriptorDigest, image.getLayers().asList().get(0).getBlobDescriptor().getDigest()); } @@ -189,18 +161,14 @@ public void test_propagateBaseImageConfiguration() .thenReturn( ImmutableSet.of( AbsoluteUnixPath.get("/new/path1"), AbsoluteUnixPath.get("/new/path2"))); - BuildImageStep buildImageStep = + Image image = new BuildImageStep( - MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()), - mockBuildConfiguration, - ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer(), - mockPullBaseImageStep, - mockPullAndCacheBaseImageLayersStep, - ImmutableList.of( - mockBuildAndCacheApplicationLayerStepDependencies, - mockBuildAndCacheApplicationLayerStepResources, - mockBuildAndCacheApplicationLayerStepClasses)); - Image image = buildImageStep.getFuture().get().getFuture().get(); + mockBuildConfiguration, + mockProgressEventDispatcherFactory, + baseImage, + baseImageLayers, + applicationLayers) + .call(); Assert.assertEquals("wasm", image.getArchitecture()); Assert.assertEquals("js", image.getOs()); Assert.assertEquals( @@ -252,18 +220,14 @@ public void testOverrideWorkingDirectory() throws InterruptedException, Executio Mockito.when(mockContainerConfiguration.getWorkingDirectory()) .thenReturn(AbsoluteUnixPath.get("/my/directory")); - BuildImageStep buildImageStep = + Image image = new BuildImageStep( - MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()), - mockBuildConfiguration, - ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer(), - mockPullBaseImageStep, - mockPullAndCacheBaseImageLayersStep, - ImmutableList.of( - mockBuildAndCacheApplicationLayerStepDependencies, - mockBuildAndCacheApplicationLayerStepResources, - mockBuildAndCacheApplicationLayerStepClasses)); - Image image = buildImageStep.getFuture().get().getFuture().get(); + mockBuildConfiguration, + mockProgressEventDispatcherFactory, + baseImage, + baseImageLayers, + applicationLayers) + .call(); Assert.assertEquals("/my/directory", image.getWorkingDirectory()); } @@ -274,18 +238,14 @@ public void test_inheritedEntrypoint() throws ExecutionException, InterruptedExc Mockito.when(mockContainerConfiguration.getProgramArguments()) .thenReturn(ImmutableList.of("test")); - BuildImageStep buildImageStep = + Image image = new BuildImageStep( - MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()), - mockBuildConfiguration, - ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer(), - mockPullBaseImageStep, - mockPullAndCacheBaseImageLayersStep, - ImmutableList.of( - mockBuildAndCacheApplicationLayerStepDependencies, - mockBuildAndCacheApplicationLayerStepResources, - mockBuildAndCacheApplicationLayerStepClasses)); - Image image = buildImageStep.getFuture().get().getFuture().get(); + mockBuildConfiguration, + mockProgressEventDispatcherFactory, + baseImage, + baseImageLayers, + applicationLayers) + .call(); Assert.assertEquals(ImmutableList.of("baseImageEntrypoint"), image.getEntrypoint()); Assert.assertEquals(ImmutableList.of("test"), image.getProgramArguments()); @@ -297,18 +257,14 @@ public void test_inheritedEntrypointAndProgramArguments() Mockito.when(mockContainerConfiguration.getEntrypoint()).thenReturn(null); Mockito.when(mockContainerConfiguration.getProgramArguments()).thenReturn(null); - BuildImageStep buildImageStep = + Image image = new BuildImageStep( - MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()), - mockBuildConfiguration, - ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer(), - mockPullBaseImageStep, - mockPullAndCacheBaseImageLayersStep, - ImmutableList.of( - mockBuildAndCacheApplicationLayerStepDependencies, - mockBuildAndCacheApplicationLayerStepResources, - mockBuildAndCacheApplicationLayerStepClasses)); - Image image = buildImageStep.getFuture().get().getFuture().get(); + mockBuildConfiguration, + mockProgressEventDispatcherFactory, + baseImage, + baseImageLayers, + applicationLayers) + .call(); Assert.assertEquals(ImmutableList.of("baseImageEntrypoint"), image.getEntrypoint()); Assert.assertEquals(ImmutableList.of("catalina.sh", "run"), image.getProgramArguments()); @@ -320,18 +276,14 @@ public void test_notInheritedProgramArguments() throws ExecutionException, Inter .thenReturn(ImmutableList.of("myEntrypoint")); Mockito.when(mockContainerConfiguration.getProgramArguments()).thenReturn(null); - BuildImageStep buildImageStep = + Image image = new BuildImageStep( - MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()), - mockBuildConfiguration, - ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer(), - mockPullBaseImageStep, - mockPullAndCacheBaseImageLayersStep, - ImmutableList.of( - mockBuildAndCacheApplicationLayerStepDependencies, - mockBuildAndCacheApplicationLayerStepResources, - mockBuildAndCacheApplicationLayerStepClasses)); - Image image = buildImageStep.getFuture().get().getFuture().get(); + mockBuildConfiguration, + mockProgressEventDispatcherFactory, + baseImage, + baseImageLayers, + applicationLayers) + .call(); Assert.assertEquals(ImmutableList.of("myEntrypoint"), image.getEntrypoint()); Assert.assertNull(image.getProgramArguments()); @@ -339,19 +291,14 @@ public void test_notInheritedProgramArguments() throws ExecutionException, Inter @Test public void test_generateHistoryObjects() throws ExecutionException, InterruptedException { - BuildImageStep buildImageStep = + Image image = new BuildImageStep( - MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()), - mockBuildConfiguration, - ProgressEventDispatcher.newRoot(mockEventHandlers, "ignored", 1).newChildProducer(), - mockPullBaseImageStep, - mockPullAndCacheBaseImageLayersStep, - ImmutableList.of( - mockBuildAndCacheApplicationLayerStepDependencies, - mockBuildAndCacheApplicationLayerStepResources, - mockBuildAndCacheApplicationLayerStepClasses, - mockBuildAndCacheApplicationLayerStepExtraFiles)); - Image image = buildImageStep.getFuture().get().getFuture().get(); + mockBuildConfiguration, + mockProgressEventDispatcherFactory, + baseImage, + baseImageLayers, + applicationLayers) + .call(); // Make sure history is as expected HistoryEntry expectedAddedBaseLayerHistory = From cdee3f6526415003f7161635cabb242e29ec8c26 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Sun, 9 Jun 2019 20:56:53 -0400 Subject: [PATCH 06/50] wip --- .../java/com/google/cloud/tools/jib/api/JibIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java index 7124df8d56..fd454c5ee8 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java @@ -145,7 +145,7 @@ public void testOffline() } catch (ExecutionException ex) { Assert.assertEquals( "Cannot run Jib in offline mode; localhost:5001/busybox not found in local Jib cache", - ex.getCause().getMessage()); + ex.getCause().getCause().getCause().getMessage()); } // Run online to cache the base image From 25a47f5948cf14b34ca35e2dc5169d2373439f98 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Sun, 9 Jun 2019 21:05:46 -0400 Subject: [PATCH 07/50] wip --- .../jib/builder/steps/AuthenticatePushStep.java | 12 ++++++------ .../tools/jib/builder/steps/PullBaseImageStep.java | 9 --------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java index d5a3ac351e..fa6b0ff7f0 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java @@ -40,15 +40,15 @@ class AuthenticatePushStep implements Callable { private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - private final Credential registryCredentials; + private final Credential registryCredential; AuthenticatePushStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, - Credential targetRegistryCredentials) { + Credential targetRegistryCredential) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; - registryCredentials = targetRegistryCredentials; + registryCredential = targetRegistryCredential; } @Override @@ -66,15 +66,15 @@ public Authorization call() throws IOException, RegistryException { .newRegistryClient() .getRegistryAuthenticator(); if (registryAuthenticator != null) { - return registryAuthenticator.authenticatePush(registryCredentials); + return registryAuthenticator.authenticatePush(registryCredential); } } catch (InsecureRegistryException ex) { // Cannot skip certificate validation or use HTTP; fall through. } - return (registryCredentials == null || registryCredentials.isOAuth2RefreshToken()) + return (registryCredential == null || registryCredential.isOAuth2RefreshToken()) ? null : Authorization.fromBasicCredentials( - registryCredentials.getUsername(), registryCredentials.getPassword()); + registryCredential.getUsername(), registryCredential.getPassword()); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index d0de2e7cca..0fbae70398 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -49,7 +49,6 @@ import com.google.cloud.tools.jib.registry.RegistryClient; import com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException; import com.google.common.annotations.VisibleForTesting; -import com.google.common.util.concurrent.ListeningExecutorService; import java.io.IOException; import java.util.Optional; import java.util.concurrent.Callable; @@ -85,14 +84,6 @@ Authorization getAuthorization() { private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - PullBaseImageStep( - ListeningExecutorService listeningExecutorService, - BuildConfiguration buildConfiguration, - ProgressEventDispatcher.Factory progressEventDispatcherFactory) { - this.buildConfiguration = buildConfiguration; - this.progressEventDispatcherFactory = progressEventDispatcherFactory; - } - PullBaseImageStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory) { From 936c7bf528d4da5ac7f4b9413e59c407d3c265d3 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 10 Jun 2019 18:52:17 -0400 Subject: [PATCH 08/50] wip --- .../jib/builder/steps/CachedLayerAndName.java | 2 +- .../tools/jib/builder/steps/PushImageStep.java | 6 +++++- .../tools/jib/builder/steps/StepsRunner.java | 7 +++++++ .../jib/builder/steps/BuildImageStepTest.java | 17 +++++++---------- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java index 85e80cdd82..eed59fc14e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java @@ -19,7 +19,7 @@ import com.google.cloud.tools.jib.cache.CachedLayer; import javax.annotation.Nullable; -/** Simple structure to hold the result pair of {#link CachedLayer} and its string name. */ +/** Simple structure to hold a pair of {#link CachedLayer} and its string name. */ class CachedLayerAndName { private CachedLayer cachedLayer; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java index 70425e0e24..7cbb0172e4 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java @@ -16,11 +16,13 @@ package com.google.cloud.tools.jib.builder.steps; +import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.api.LogEvent; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; @@ -107,7 +109,9 @@ public BuildResult call() throws IOException, InterruptedException, ExecutionExc })); } - BuildResult result = BuildResult.fromImage(builtImage, buildConfiguration.getTargetFormat()); + DescriptorDigest imageDigest = Digests.computeJsonDigest(manifestTemplate); + DescriptorDigest imageId = containerConfigurationDigestAndSize.getDigest(); + BuildResult result = new BuildResult(imageDigest, imageId); return Futures.whenAllSucceed(pushAllTagsFutures) .call( diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 6293784860..16b1b47d4c 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -91,6 +91,13 @@ public static StepsRunner begin(BuildConfiguration buildConfiguration) { private final ListeningExecutorService executorService; private final BuildConfiguration buildConfiguration; + // We save steps to run by wrapping each step into a Runnable, only because of the unfortunate + // chicken-and-egg situation arising from using ProgressEventDispatcher. The current + // ProgressEventDispatcher model requires knowing in advance how many units of work (i.e., steps) + // we should perform. That is, to instantiate a root ProgressEventDispatcher instance, we should + // know ahead how many steps we will run. However, to instantiate a step, we need a root progress + // dispatcher. So, we wrap steps into Runnables and save them to run them later. Then we can count + // the number of Runnables and, create a root dispatcher, and run the saved Runnables. private final List stepsToRun = new ArrayList<>(); @Nullable private String rootProgressDescription; diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java index d35d4b7c30..96b65640fa 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java @@ -37,7 +37,6 @@ import java.time.Instant; import java.util.Arrays; import java.util.List; -import java.util.concurrent.ExecutionException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -135,7 +134,7 @@ public void setUp() throws DigestException { } @Test - public void test_validateAsyncDependencies() throws ExecutionException, InterruptedException { + public void test_validateAsyncDependencies() { Image image = new BuildImageStep( mockBuildConfiguration, @@ -149,8 +148,7 @@ public void test_validateAsyncDependencies() throws ExecutionException, Interrup } @Test - public void test_propagateBaseImageConfiguration() - throws ExecutionException, InterruptedException { + public void test_propagateBaseImageConfiguration() { Mockito.when(mockContainerConfiguration.getEnvironmentMap()) .thenReturn(ImmutableMap.of("MY_ENV", "MY_ENV_VALUE", "BASE_ENV_2", "NEW_VALUE")); Mockito.when(mockContainerConfiguration.getLabels()) @@ -216,7 +214,7 @@ public void test_propagateBaseImageConfiguration() } @Test - public void testOverrideWorkingDirectory() throws InterruptedException, ExecutionException { + public void testOverrideWorkingDirectory() { Mockito.when(mockContainerConfiguration.getWorkingDirectory()) .thenReturn(AbsoluteUnixPath.get("/my/directory")); @@ -233,7 +231,7 @@ public void testOverrideWorkingDirectory() throws InterruptedException, Executio } @Test - public void test_inheritedEntrypoint() throws ExecutionException, InterruptedException { + public void test_inheritedEntrypoint() { Mockito.when(mockContainerConfiguration.getEntrypoint()).thenReturn(null); Mockito.when(mockContainerConfiguration.getProgramArguments()) .thenReturn(ImmutableList.of("test")); @@ -252,8 +250,7 @@ public void test_inheritedEntrypoint() throws ExecutionException, InterruptedExc } @Test - public void test_inheritedEntrypointAndProgramArguments() - throws ExecutionException, InterruptedException { + public void test_inheritedEntrypointAndProgramArguments() { Mockito.when(mockContainerConfiguration.getEntrypoint()).thenReturn(null); Mockito.when(mockContainerConfiguration.getProgramArguments()).thenReturn(null); @@ -271,7 +268,7 @@ public void test_inheritedEntrypointAndProgramArguments() } @Test - public void test_notInheritedProgramArguments() throws ExecutionException, InterruptedException { + public void test_notInheritedProgramArguments() { Mockito.when(mockContainerConfiguration.getEntrypoint()) .thenReturn(ImmutableList.of("myEntrypoint")); Mockito.when(mockContainerConfiguration.getProgramArguments()).thenReturn(null); @@ -290,7 +287,7 @@ public void test_notInheritedProgramArguments() throws ExecutionException, Inter } @Test - public void test_generateHistoryObjects() throws ExecutionException, InterruptedException { + public void test_generateHistoryObjects() { Image image = new BuildImageStep( mockBuildConfiguration, From 507b9d759275b888286bb698b41a339e724c3b19 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 11 Jun 2019 12:03:48 -0400 Subject: [PATCH 09/50] Unwrap nested execution exception --- .../jib/plugins/common/JibBuildRunner.java | 8 +++- .../plugins/common/JibBuildRunnerTest.java | 37 ++++++++++++++----- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunner.java b/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunner.java index e55926a19c..8f36eedab2 100644 --- a/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunner.java +++ b/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunner.java @@ -228,8 +228,12 @@ public JibContainer build( throw new BuildStepsExecutionException(message, ex); } catch (ExecutionException ex) { - String message = Verify.verifyNotNull(ex.getCause().getMessage()); // keep null-away happy - throw new BuildStepsExecutionException(message, ex.getCause()); + Throwable throwable = ex; + while (throwable instanceof ExecutionException && throwable.getCause() != null) { + throwable = throwable.getCause(); + } + String message = Verify.verifyNotNull(throwable.getMessage()); // keep null-away happy + throw new BuildStepsExecutionException(message, throwable); } catch (InterruptedException ex) { // TODO: Add more suggestions for various build failures. diff --git a/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunnerTest.java b/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunnerTest.java index 4f1135bbf3..d042adbb41 100644 --- a/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunnerTest.java +++ b/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunnerTest.java @@ -91,7 +91,7 @@ public void testBuildImage_httpHostConnectException() try { testJibBuildRunner.build( mockJibContainerBuilder, mockContainerizer, logEvent -> {}, TEST_HELPFUL_SUGGESTIONS); - Assert.fail("buildImage should have thrown an exception"); + Assert.fail(); } catch (BuildStepsExecutionException ex) { Assert.assertEquals(TEST_HELPFUL_SUGGESTIONS.forHttpHostConnect(), ex.getMessage()); @@ -110,7 +110,7 @@ public void testBuildImage_unknownHostException() try { testJibBuildRunner.build( mockJibContainerBuilder, mockContainerizer, logEvent -> {}, TEST_HELPFUL_SUGGESTIONS); - Assert.fail("buildImage should have thrown an exception"); + Assert.fail(); } catch (BuildStepsExecutionException ex) { Assert.assertEquals(TEST_HELPFUL_SUGGESTIONS.forUnknownHost(), ex.getMessage()); @@ -130,7 +130,7 @@ public void testBuildImage_insecureRegistryException() try { testJibBuildRunner.build( mockJibContainerBuilder, mockContainerizer, logEvent -> {}, TEST_HELPFUL_SUGGESTIONS); - Assert.fail("buildImage should have thrown an exception"); + Assert.fail(); } catch (BuildStepsExecutionException ex) { Assert.assertEquals(TEST_HELPFUL_SUGGESTIONS.forInsecureRegistry(), ex.getMessage()); @@ -155,7 +155,7 @@ public void testBuildImage_registryUnauthorizedException_statusCodeForbidden() try { testJibBuildRunner.build( mockJibContainerBuilder, mockContainerizer, logEvent -> {}, TEST_HELPFUL_SUGGESTIONS); - Assert.fail("buildImage should have thrown an exception"); + Assert.fail(); } catch (BuildStepsExecutionException ex) { Assert.assertEquals( @@ -181,7 +181,7 @@ public void testBuildImage_registryUnauthorizedException_noCredentials() try { testJibBuildRunner.build( mockJibContainerBuilder, mockContainerizer, logEvent -> {}, TEST_HELPFUL_SUGGESTIONS); - Assert.fail("buildImage should have thrown an exception"); + Assert.fail(); } catch (BuildStepsExecutionException ex) { Assert.assertEquals( @@ -201,7 +201,7 @@ public void testBuildImage_registryCredentialsNotSentException() try { testJibBuildRunner.build( mockJibContainerBuilder, mockContainerizer, logEvent -> {}, TEST_HELPFUL_SUGGESTIONS); - Assert.fail("buildImage should have thrown an exception"); + Assert.fail(); } catch (BuildStepsExecutionException ex) { Assert.assertEquals(TEST_HELPFUL_SUGGESTIONS.forCredentialsNotSent(), ex.getMessage()); @@ -212,18 +212,37 @@ public void testBuildImage_registryCredentialsNotSentException() public void testBuildImage_other() throws InterruptedException, IOException, CacheDirectoryCreationException, RegistryException, ExecutionException { - RegistryException registryException = new RegistryException("messagePrefix"); - Mockito.doThrow(registryException) + Mockito.doThrow(new RegistryException("messagePrefix")) .when(mockJibContainerBuilder) .containerize(mockContainerizer); try { testJibBuildRunner.build( mockJibContainerBuilder, mockContainerizer, logEvent -> {}, TEST_HELPFUL_SUGGESTIONS); - Assert.fail("buildImage should have thrown an exception"); + Assert.fail(); } catch (BuildStepsExecutionException ex) { Assert.assertEquals(TEST_HELPFUL_SUGGESTIONS.none(), ex.getMessage()); } } + + @Test + public void testBuildImage_nestedExecutionExceptions() + throws InterruptedException, RegistryException, IOException, CacheDirectoryCreationException, + ExecutionException { + Mockito.doThrow( + new ExecutionException( + new ExecutionException(new ExecutionException(new IOException("awesome"))))) + .when(mockJibContainerBuilder) + .containerize(mockContainerizer); + + try { + testJibBuildRunner.build( + mockJibContainerBuilder, mockContainerizer, logEvent -> {}, TEST_HELPFUL_SUGGESTIONS); + Assert.fail(); + + } catch (BuildStepsExecutionException ex) { + Assert.assertEquals("awesome", ex.getMessage()); + } + } } From 74020efe62a97d8510f87a4080faa34fc4deda18 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 11 Jun 2019 16:30:59 -0400 Subject: [PATCH 10/50] Refactor PushImageStep --- .../BuildAndCacheApplicationLayerStep.java | 4 +- .../jib/builder/steps/PushImageStep.java | 129 +++++++++--------- .../tools/jib/builder/steps/StepsRunner.java | 26 ++-- .../jib/image/json/ImageToJsonTranslator.java | 13 +- .../jib/builder/steps/BuildImageStepTest.java | 2 +- 5 files changed, 82 insertions(+), 92 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java index 4a4de43f40..05d88cae13 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java @@ -65,9 +65,7 @@ static ImmutableList makeList( layerConfiguration.getName(), layerConfiguration)); } - ImmutableList steps = - buildAndCacheApplicationLayerSteps.build(); - return steps; + return buildAndCacheApplicationLayerSteps.build(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java index 7cbb0172e4..3b063d2f6f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java @@ -18,6 +18,7 @@ import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.api.LogEvent; +import com.google.cloud.tools.jib.api.RegistryException; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; @@ -28,99 +29,99 @@ import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; import com.google.cloud.tools.jib.registry.RegistryClient; -import com.google.common.collect.ImmutableSet; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.collect.ImmutableList; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.util.Set; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -/** Pushes the final image. Outputs the pushed image digest. */ +/** + * Pushes the final image. In fact, its manifest is pushed along with a tag. Returns the pushed + * image digest (digest of manifest) and image ID (digest of container configuration) as {#link + * BuildResult}. + */ class PushImageStep implements Callable { - private static final String DESCRIPTION = "Pushing new image"; + private static final String DESCRIPTION = "Pushing new image (tag)"; private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; + private final BuildableManifestTemplate manifestTemplate; private final Authorization pushAuthorization; - private final BlobDescriptor containerConfigurationDigestAndSize; - private final Image builtImage; - - private final ListeningExecutorService listeningExecutorService; + private final String tag; + private final DescriptorDigest imageDigest; + private final DescriptorDigest imageId; - // TODO: remove listeningExecutorService like other siblings PushImageStep( - ListeningExecutorService listeningExecutorService, BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Authorization pushAuthorization, - BlobDescriptor containerConfigurationDigestAndSize, - Image builtImage) { - this.listeningExecutorService = listeningExecutorService; + BuildableManifestTemplate manifestTemplate, + String tag, + DescriptorDigest imageDigest, + DescriptorDigest imageId) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.pushAuthorization = pushAuthorization; - this.containerConfigurationDigestAndSize = containerConfigurationDigestAndSize; - this.builtImage = builtImage; + this.manifestTemplate = manifestTemplate; + this.tag = tag; + this.imageDigest = imageDigest; + this.imageId = imageId; } - @Override - public BuildResult call() throws IOException, InterruptedException, ExecutionException { - ImmutableSet targetImageTags = buildConfiguration.getAllTargetImageTags(); - ProgressEventDispatcher progressEventDispatcher = - progressEventDispatcherFactory.create("pushing image manifest", targetImageTags.size()); + static ImmutableList makeList( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, + Authorization pushAuthorization, + BlobDescriptor containerConfigurationDigestAndSize, + Image builtImage) + throws IOException { + Set tags = buildConfiguration.getAllTargetImageTags(); try (TimerEventDispatcher ignored = - new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { - RegistryClient registryClient = - buildConfiguration - .newTargetImageRegistryClientFactory() - .setAuthorization(pushAuthorization) - .newRegistryClient(); - - // Constructs the image. - ImageToJsonTranslator imageToJsonTranslator = new ImageToJsonTranslator(builtImage); + new TimerEventDispatcher( + buildConfiguration.getEventHandlers(), "Preparing image (tag) pushers"); + ProgressEventDispatcher progressEventDispatcher = + progressEventDispatcherFactory.create("preparing (image) tag pushers", tags.size())) { // Gets the image manifest to push. BuildableManifestTemplate manifestTemplate = - imageToJsonTranslator.getManifestTemplate( - buildConfiguration.getTargetFormat(), containerConfigurationDigestAndSize); + new ImageToJsonTranslator(builtImage) + .getManifestTemplate( + buildConfiguration.getTargetFormat(), containerConfigurationDigestAndSize); - // Pushes to all target image tags. - List> pushAllTagsFutures = new ArrayList<>(); - for (String tag : targetImageTags) { - ProgressEventDispatcher.Factory progressEventDispatcherFactory = - progressEventDispatcher.newChildProducer(); - pushAllTagsFutures.add( - listeningExecutorService.submit( - () -> { - try (ProgressEventDispatcher ignored2 = - progressEventDispatcherFactory.create("tagging with " + tag, 1)) { - buildConfiguration - .getEventHandlers() - .dispatch(LogEvent.info("Tagging with " + tag + "...")); - registryClient.pushManifest(manifestTemplate, tag); - } - return null; - })); + ImmutableList.Builder imagePushers = new ImmutableList.Builder<>(); + for (String tag : buildConfiguration.getAllTargetImageTags()) { + imagePushers.add( + new PushImageStep( + buildConfiguration, + progressEventDispatcher.newChildProducer(), + pushAuthorization, + manifestTemplate, + tag, + Digests.computeJsonDigest(manifestTemplate), + containerConfigurationDigestAndSize.getDigest())); } + return imagePushers.build(); + } + } + + @Override + public BuildResult call() throws IOException, RegistryException { + try (TimerEventDispatcher ignored = + new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION); + ProgressEventDispatcher ignored2 = + progressEventDispatcherFactory.create("tagging with " + tag, 1)) { + buildConfiguration.getEventHandlers().dispatch(LogEvent.info("Tagging with " + tag + "...")); - DescriptorDigest imageDigest = Digests.computeJsonDigest(manifestTemplate); - DescriptorDigest imageId = containerConfigurationDigestAndSize.getDigest(); - BuildResult result = new BuildResult(imageDigest, imageId); + RegistryClient registryClient = + buildConfiguration + .newTargetImageRegistryClientFactory() + .setAuthorization(pushAuthorization) + .newRegistryClient(); - return Futures.whenAllSucceed(pushAllTagsFutures) - .call( - () -> { - progressEventDispatcher.close(); - return result; - }, - listeningExecutorService) - .get(); + registryClient.pushManifest(manifestTemplate, tag); + return new BuildResult(imageDigest, imageId); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 16b1b47d4c..4e055bb4d4 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -87,8 +87,7 @@ public static StepsRunner begin(BuildConfiguration buildConfiguration) { private final StepResults results = new StepResults(); - // TODO: use plain ExecutorService; requires refactoring PushImageStep. - private final ListeningExecutorService executorService; + private final ExecutorService executorService; private final BuildConfiguration buildConfiguration; // We save steps to run by wrapping each step into a Runnable, only because of the unfortunate @@ -204,21 +203,24 @@ private void pushApplicationLayers() { results.applicationLayers))); } - private void pushImage() { + private void pushImages() { results.buildResult = executorService.submit( () -> { realizeFutures(results.baseImageLayerPushResults.get()); realizeFutures(results.applicationLayerPushResults.get()); - return new PushImageStep( - executorService, - buildConfiguration, - childProgressDispatcherFactorySupplier.get(), - results.pushAuthorization.get(), - results.containerConfigurationPushResult.get(), - results.builtImage.get()) - .call(); + List> tagPushResults = + scheduleCallables( + PushImageStep.makeList( + buildConfiguration, + childProgressDispatcherFactorySupplier.get(), + results.pushAuthorization.get(), + results.containerConfigurationPushResult.get(), + results.builtImage.get())); + realizeFutures(tagPushResults); + // Image (tag, or actually manifest) pushers return the same BuildResult. + return tagPushResults.get(0).get(); }); } @@ -300,7 +302,7 @@ public StepsRunner registryPushSteps() { stepsToRun.add(this::pushBaseImageLayers); stepsToRun.add(this::pushApplicationLayers); stepsToRun.add(this::pushContainerConfiguration); - stepsToRun.add(this::pushImage); + stepsToRun.add(this::pushImages); return this; } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java index 407c333348..3f903b89f3 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java @@ -36,18 +36,7 @@ import java.util.function.Function; import javax.annotation.Nullable; -/** - * Translates an {@link Image} into a manifest or container configuration JSON BLOB. - * - *

Example usage: - * - * 

{@code
- * ImageToJsonTranslator translator = new ImageToJsonTranslator(image);
- * Blob containerConfigurationBlob = translator.getContainerConfigurationBlob();
- * BlobDescriptor containerConfigurationBlobDescriptor = blob.writeTo(outputStream);
- * Blob manifestBlob = translator.getManifestBlob(containerConfigurationBlobDescriptor);
- * }
- */ +/** Translates an {@link Image} into a manifest or container configuration JSON BLOB. */ public class ImageToJsonTranslator { /** diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java index 96b65640fa..5fe9044450 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java @@ -134,7 +134,7 @@ public void setUp() throws DigestException { } @Test - public void test_validateAsyncDependencies() { + public void test_basicCase() { Image image = new BuildImageStep( mockBuildConfiguration, From 398f50fe7be5b2c2a5b2e6434773f862a3fdbc90 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 11 Jun 2019 16:43:23 -0400 Subject: [PATCH 11/50] Remove PullAndCacheBaseImageLayersStep --- .../BuildAndCacheApplicationLayerStep.java | 7 +- .../steps/PullAndCacheBaseImageLayerStep.java | 31 +++++ .../PullAndCacheBaseImageLayersStep.java | 59 ---------- .../jib/builder/steps/PushImageStep.java | 52 ++++----- .../jib/builder/steps/PushLayerStep.java | 19 ++- .../tools/jib/builder/steps/StepsRunner.java | 110 +++++++++--------- 6 files changed, 125 insertions(+), 153 deletions(-) delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java index 05d88cae13..f282347c04 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java @@ -34,7 +34,7 @@ /** Builds and caches application layers. */ class BuildAndCacheApplicationLayerStep implements Callable { - private static final String DESCRIPTION = "Preparing application layer builders"; + private static final String DESCRIPTION = "Building %s layer"; /** * Makes a list of {@link BuildAndCacheApplicationLayerStep} for dependencies, resources, and @@ -49,7 +49,8 @@ static ImmutableList makeList( progressEventDispatcherFactory.create( "preparing application layer builders", layerCount); TimerEventDispatcher ignored = - new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { + new TimerEventDispatcher( + buildConfiguration.getEventHandlers(), "Preparing application layer builders")) { ImmutableList.Builder buildAndCacheApplicationLayerSteps = ImmutableList.builderWithExpectedSize(layerCount); for (LayerConfiguration layerConfiguration : buildConfiguration.getLayerConfigurations()) { @@ -88,7 +89,7 @@ private BuildAndCacheApplicationLayerStep( @Override public CachedLayerAndName call() throws IOException, CacheCorruptedException { - String description = "Building " + layerName + " layer"; + String description = String.format(DESCRIPTION, layerName); buildConfiguration.getEventHandlers().dispatch(LogEvent.progress(description + "...")); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 7fbf042b75..0868aef0ac 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -19,13 +19,18 @@ import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; +import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; import com.google.cloud.tools.jib.cache.Cache; import com.google.cloud.tools.jib.cache.CacheCorruptedException; import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.http.Authorization; +import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.registry.RegistryClient; +import com.google.common.collect.ImmutableList; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.concurrent.Callable; import javax.annotation.Nullable; @@ -35,6 +40,32 @@ class PullAndCacheBaseImageLayerStep implements Callable { private static final String DESCRIPTION = "Pulling base image layer %s"; + static ImmutableList makeList( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, + ImageAndAuthorization baseImageAndAuth) { + ImmutableList baseImageLayers = baseImageAndAuth.getImage().getLayers(); + + try (ProgressEventDispatcher progressEventDispatcher = + progressEventDispatcherFactory.create( + "preparing base image layer pullers", baseImageLayers.size()); + TimerEventDispatcher ignored = + new TimerEventDispatcher( + buildConfiguration.getEventHandlers(), "Preparing base image layer pullers")) { + + List layerPullers = new ArrayList<>(); + for (Layer layer : baseImageLayers) { + layerPullers.add( + new PullAndCacheBaseImageLayerStep( + buildConfiguration, + progressEventDispatcher.newChildProducer(), + layer.getBlobDescriptor().getDigest(), + baseImageAndAuth.getAuthorization())); + } + return ImmutableList.copyOf(layerPullers); + } + } + private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java deleted file mode 100644 index 1ade0d839e..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.builder.steps; - -import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; -import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; -import com.google.cloud.tools.jib.configuration.BuildConfiguration; -import com.google.cloud.tools.jib.image.Layer; -import com.google.common.collect.ImmutableList; -import java.util.ArrayList; -import java.util.List; - -/** Pulls and caches the base image layers. */ -// TODO: following the same pattern as "BuildAndCacheApplicationLayerStep", move the sole "makeList" -// static into "PullAndCacheBaseImageLayerStep" and remove this class. -class PullAndCacheBaseImageLayersStep { - - private static final String DESCRIPTION = "Preparing base image layer pullers"; - - static ImmutableList makeList( - BuildConfiguration buildConfiguration, - ProgressEventDispatcher.Factory progressEventDispatcherFactory, - ImageAndAuthorization baseImageAndAuth) { - ImmutableList baseImageLayers = baseImageAndAuth.getImage().getLayers(); - - try (ProgressEventDispatcher progressEventDispatcher = - progressEventDispatcherFactory.create( - "preparing base image layer pullers", baseImageLayers.size()); - TimerEventDispatcher ignored = - new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { - - List layerPullers = new ArrayList<>(); - for (Layer layer : baseImageLayers) { - layerPullers.add( - new PullAndCacheBaseImageLayerStep( - buildConfiguration, - progressEventDispatcher.newChildProducer(), - layer.getBlobDescriptor().getDigest(), - baseImageAndAuth.getAuthorization())); - } - return ImmutableList.copyOf(layerPullers); - } - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java index 3b063d2f6f..142b1e2014 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java @@ -43,32 +43,6 @@ class PushImageStep implements Callable { private static final String DESCRIPTION = "Pushing new image (tag)"; - private final BuildConfiguration buildConfiguration; - private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - - private final BuildableManifestTemplate manifestTemplate; - private final Authorization pushAuthorization; - private final String tag; - private final DescriptorDigest imageDigest; - private final DescriptorDigest imageId; - - PushImageStep( - BuildConfiguration buildConfiguration, - ProgressEventDispatcher.Factory progressEventDispatcherFactory, - Authorization pushAuthorization, - BuildableManifestTemplate manifestTemplate, - String tag, - DescriptorDigest imageDigest, - DescriptorDigest imageId) { - this.buildConfiguration = buildConfiguration; - this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.pushAuthorization = pushAuthorization; - this.manifestTemplate = manifestTemplate; - this.tag = tag; - this.imageDigest = imageDigest; - this.imageId = imageId; - } - static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, @@ -106,6 +80,32 @@ static ImmutableList makeList( } } + PushImageStep( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, + Authorization pushAuthorization, + BuildableManifestTemplate manifestTemplate, + String tag, + DescriptorDigest imageDigest, + DescriptorDigest imageId) { + this.buildConfiguration = buildConfiguration; + this.progressEventDispatcherFactory = progressEventDispatcherFactory; + this.pushAuthorization = pushAuthorization; + this.manifestTemplate = manifestTemplate; + this.tag = tag; + this.imageDigest = imageDigest; + this.imageId = imageId; + } + + private final BuildConfiguration buildConfiguration; + private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; + + private final BuildableManifestTemplate manifestTemplate; + private final Authorization pushAuthorization; + private final String tag; + private final DescriptorDigest imageDigest; + private final DescriptorDigest imageId; + @Override public BuildResult call() throws IOException, RegistryException { try (TimerEventDispatcher ignored = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java index 26e9b552b8..378bb738e6 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java @@ -33,24 +33,17 @@ class PushLayerStep implements Callable { - private static final String DESCRIPTION = "Preparing application layer pushers"; - - private final BuildConfiguration buildConfiguration; - private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - - private final Authorization pushAuthorization; - private final Future cachedLayerAndName; - static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Authorization pushAuthorization, List> cachedLayers) { try (TimerEventDispatcher ignored = - new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION); + new TimerEventDispatcher( + buildConfiguration.getEventHandlers(), "Preparing application layer pushers"); ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create( - "Preparing application layer pushers", cachedLayers.size())) { + "preparing application layer pushers", cachedLayers.size())) { // Constructs a PushBlobStep for each layer. List blobPushers = new ArrayList<>(); @@ -65,6 +58,12 @@ static ImmutableList makeList( } } + private final BuildConfiguration buildConfiguration; + private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; + + private final Authorization pushAuthorization; + private final Future cachedLayerAndName; + PushLayerStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 4e055bb4d4..993a95dcfc 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -85,6 +85,15 @@ public static StepsRunner begin(BuildConfiguration buildConfiguration) { return new StepsRunner(MoreExecutors.listeningDecorator(executorService), buildConfiguration); } + private static List realizeFutures(List> futures) + throws InterruptedException, ExecutionException { + List values = new ArrayList<>(); + for (Future future : futures) { + values.add(future.get()); + } + return values; + } + private final StepResults results = new StepResults(); private final ExecutorService executorService; @@ -111,6 +120,51 @@ private StepsRunner( this.buildConfiguration = buildConfiguration; } + public StepsRunner dockerLoadSteps(DockerClient dockerClient) { + rootProgressDescription = "building image to Docker daemon"; + enqueueBuildAndCache(); + stepsToRun.add(() -> loadDocker(dockerClient)); + return this; + } + + public StepsRunner tarBuildSteps(Path outputPath) { + rootProgressDescription = "building image to tar file"; + enqueueBuildAndCache(); + stepsToRun.add(() -> writeTarFile(outputPath)); + return this; + } + + public StepsRunner registryPushSteps() { + rootProgressDescription = "building image to registry"; + enqueueBuildAndCache(); + stepsToRun.add(this::retrieveTargetRegistryCredentials); + stepsToRun.add(this::authenticatePush); + stepsToRun.add(this::pushBaseImageLayers); + stepsToRun.add(this::pushApplicationLayers); + stepsToRun.add(this::pushContainerConfiguration); + stepsToRun.add(this::pushImages); + return this; + } + + public BuildResult run() throws ExecutionException, InterruptedException { + Preconditions.checkNotNull(rootProgressDescription); + + try (ProgressEventDispatcher progressEventDispatcher = + ProgressEventDispatcher.newRoot( + buildConfiguration.getEventHandlers(), rootProgressDescription, stepsToRun.size())) { + childProgressDispatcherFactorySupplier = progressEventDispatcher::newChildProducer; + stepsToRun.forEach(Runnable::run); + return results.buildResult.get(); + } + } + + private void enqueueBuildAndCache() { + stepsToRun.add(this::pullBaseImage); + stepsToRun.add(this::pullAndCacheBaseImageLayers); + stepsToRun.add(this::buildAndCacheApplicationLayers); + stepsToRun.add(this::buildImage); + } + private void retrieveTargetRegistryCredentials() { results.targetRegistryCredentials = executorService.submit( @@ -141,7 +195,7 @@ private void pullAndCacheBaseImageLayers() { executorService.submit( () -> scheduleCallables( - PullAndCacheBaseImageLayersStep.makeList( + PullAndCacheBaseImageLayerStep.makeList( buildConfiguration, childProgressDispatcherFactorySupplier.get(), results.baseImageAndAuth.get()))); @@ -248,61 +302,7 @@ private void writeTarFile(Path outputPath) { .call()); } - public BuildResult run() throws ExecutionException, InterruptedException { - Preconditions.checkNotNull(rootProgressDescription); - - try (ProgressEventDispatcher progressEventDispatcher = - ProgressEventDispatcher.newRoot( - buildConfiguration.getEventHandlers(), rootProgressDescription, stepsToRun.size())) { - childProgressDispatcherFactorySupplier = progressEventDispatcher::newChildProducer; - stepsToRun.forEach(Runnable::run); - return results.buildResult.get(); - } - } - - private static List realizeFutures(List> futures) - throws InterruptedException, ExecutionException { - List values = new ArrayList<>(); - for (Future future : futures) { - values.add(future.get()); - } - return values; - } - private List> scheduleCallables(ImmutableList> callables) { return callables.stream().map(executorService::submit).collect(Collectors.toList()); } - - private void enqueueBuildAndCache() { - stepsToRun.add(this::pullBaseImage); - stepsToRun.add(this::pullAndCacheBaseImageLayers); - stepsToRun.add(this::buildAndCacheApplicationLayers); - stepsToRun.add(this::buildImage); - } - - public StepsRunner dockerLoadSteps(DockerClient dockerClient) { - rootProgressDescription = "building image to Docker daemon"; - enqueueBuildAndCache(); - stepsToRun.add(() -> loadDocker(dockerClient)); - return this; - } - - public StepsRunner tarBuildSteps(Path outputPath) { - rootProgressDescription = "building image to tar file"; - enqueueBuildAndCache(); - stepsToRun.add(() -> writeTarFile(outputPath)); - return this; - } - - public StepsRunner registryPushSteps() { - rootProgressDescription = "building image to registry"; - enqueueBuildAndCache(); - stepsToRun.add(this::retrieveTargetRegistryCredentials); - stepsToRun.add(this::authenticatePush); - stepsToRun.add(this::pushBaseImageLayers); - stepsToRun.add(this::pushApplicationLayers); - stepsToRun.add(this::pushContainerConfiguration); - stepsToRun.add(this::pushImages); - return this; - } } From a1bf0c28015621e468b88fb2ad45336432b5e337 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 11 Jun 2019 17:23:52 -0400 Subject: [PATCH 12/50] variable renaming and minor changes --- .../java/com/google/cloud/tools/jib/api/Containerizer.java | 5 ++--- .../cloud/tools/jib/builder/steps/AuthenticatePushStep.java | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java index c9526c08b0..cea467bfbb 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/api/Containerizer.java @@ -284,9 +284,8 @@ Path getApplicationLayersCacheDirectory() throws CacheDirectoryCreationException if (applicationLayersCacheDirectory == null) { // Uses a temporary directory if application layers cache directory is not set. try { - Path temporaryDirectory = Files.createTempDirectory(null); - temporaryDirectory.toFile().deleteOnExit(); - applicationLayersCacheDirectory = temporaryDirectory; + applicationLayersCacheDirectory = Files.createTempDirectory(null); + applicationLayersCacheDirectory.toFile().deleteOnExit(); } catch (IOException ex) { throw new CacheDirectoryCreationException(ex); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java index fa6b0ff7f0..2510c5f3d5 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/AuthenticatePushStep.java @@ -45,10 +45,10 @@ class AuthenticatePushStep implements Callable { AuthenticatePushStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, - Credential targetRegistryCredential) { + Credential registryCredential) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; - registryCredential = targetRegistryCredential; + this.registryCredential = registryCredential; } @Override From 5c71c1d4a1d839eea1b7c516caa1e6983e7b8a3e Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 11 Jun 2019 17:28:08 -0400 Subject: [PATCH 13/50] Duplicate small section of code --- .../tools/jib/builder/steps/StepsRunner.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 16b1b47d4c..2bfb5d8a28 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -271,30 +271,38 @@ private List> scheduleCallables(ImmutableList loadDocker(dockerClient)); return this; } public StepsRunner tarBuildSteps(Path outputPath) { rootProgressDescription = "building image to tar file"; - enqueueBuildAndCache(); + // Phase 1: build and cache + stepsToRun.add(this::pullBaseImage); + stepsToRun.add(this::pullAndCacheBaseImageLayers); + stepsToRun.add(this::buildAndCacheApplicationLayers); + stepsToRun.add(this::buildImage); + // Phase 2: create a tar stepsToRun.add(() -> writeTarFile(outputPath)); return this; } public StepsRunner registryPushSteps() { rootProgressDescription = "building image to registry"; - enqueueBuildAndCache(); + // Phase 1: build and cache + stepsToRun.add(this::pullBaseImage); + stepsToRun.add(this::pullAndCacheBaseImageLayers); + stepsToRun.add(this::buildAndCacheApplicationLayers); + stepsToRun.add(this::buildImage); + // Phase 2: push to registry stepsToRun.add(this::retrieveTargetRegistryCredentials); stepsToRun.add(this::authenticatePush); stepsToRun.add(this::pushBaseImageLayers); From 2853f533503313dea6dd1ed55a79842cf6f4efab Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 11 Jun 2019 17:51:31 -0400 Subject: [PATCH 14/50] Unroll nested ExecutionException --- .../tools/jib/api/JibIntegrationTest.java | 2 +- .../tools/jib/builder/steps/StepsRunner.java | 19 ++++++++++++------ .../jib/plugins/common/JibBuildRunner.java | 8 ++------ .../plugins/common/JibBuildRunnerTest.java | 20 ------------------- 4 files changed, 16 insertions(+), 33 deletions(-) diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java index fd454c5ee8..7124df8d56 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java @@ -145,7 +145,7 @@ public void testOffline() } catch (ExecutionException ex) { Assert.assertEquals( "Cannot run Jib in offline mode; localhost:5001/busybox not found in local Jib cache", - ex.getCause().getCause().getCause().getMessage()); + ex.getCause().getMessage()); } // Run online to cache the base image diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 2bfb5d8a28..8f18f760b9 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -255,6 +255,13 @@ public BuildResult run() throws ExecutionException, InterruptedException { childProgressDispatcherFactorySupplier = progressEventDispatcher::newChildProducer; stepsToRun.forEach(Runnable::run); return results.buildResult.get(); + + } catch (ExecutionException ex) { + ExecutionException unrolled = ex; + while (unrolled.getCause() instanceof ExecutionException) { + unrolled = (ExecutionException) unrolled.getCause(); + } + throw unrolled; } } @@ -273,36 +280,36 @@ private List> scheduleCallables(ImmutableList loadDocker(dockerClient)); return this; } public StepsRunner tarBuildSteps(Path outputPath) { rootProgressDescription = "building image to tar file"; - // Phase 1: build and cache + // build and cache stepsToRun.add(this::pullBaseImage); stepsToRun.add(this::pullAndCacheBaseImageLayers); stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); - // Phase 2: create a tar + // create a tar stepsToRun.add(() -> writeTarFile(outputPath)); return this; } public StepsRunner registryPushSteps() { rootProgressDescription = "building image to registry"; - // Phase 1: build and cache + // build and cache stepsToRun.add(this::pullBaseImage); stepsToRun.add(this::pullAndCacheBaseImageLayers); stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); - // Phase 2: push to registry + // push to registry stepsToRun.add(this::retrieveTargetRegistryCredentials); stepsToRun.add(this::authenticatePush); stepsToRun.add(this::pushBaseImageLayers); diff --git a/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunner.java b/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunner.java index 8f36eedab2..e55926a19c 100644 --- a/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunner.java +++ b/jib-plugins-common/src/main/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunner.java @@ -228,12 +228,8 @@ public JibContainer build( throw new BuildStepsExecutionException(message, ex); } catch (ExecutionException ex) { - Throwable throwable = ex; - while (throwable instanceof ExecutionException && throwable.getCause() != null) { - throwable = throwable.getCause(); - } - String message = Verify.verifyNotNull(throwable.getMessage()); // keep null-away happy - throw new BuildStepsExecutionException(message, throwable); + String message = Verify.verifyNotNull(ex.getCause().getMessage()); // keep null-away happy + throw new BuildStepsExecutionException(message, ex.getCause()); } catch (InterruptedException ex) { // TODO: Add more suggestions for various build failures. diff --git a/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunnerTest.java b/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunnerTest.java index d042adbb41..f5e775609d 100644 --- a/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunnerTest.java +++ b/jib-plugins-common/src/test/java/com/google/cloud/tools/jib/plugins/common/JibBuildRunnerTest.java @@ -225,24 +225,4 @@ public void testBuildImage_other() Assert.assertEquals(TEST_HELPFUL_SUGGESTIONS.none(), ex.getMessage()); } } - - @Test - public void testBuildImage_nestedExecutionExceptions() - throws InterruptedException, RegistryException, IOException, CacheDirectoryCreationException, - ExecutionException { - Mockito.doThrow( - new ExecutionException( - new ExecutionException(new ExecutionException(new IOException("awesome"))))) - .when(mockJibContainerBuilder) - .containerize(mockContainerizer); - - try { - testJibBuildRunner.build( - mockJibContainerBuilder, mockContainerizer, logEvent -> {}, TEST_HELPFUL_SUGGESTIONS); - Assert.fail(); - - } catch (BuildStepsExecutionException ex) { - Assert.assertEquals("awesome", ex.getMessage()); - } - } } From 2a617f2b59f6f6af9328d4e4ffb70a1b0265963d Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 13 Jun 2019 17:15:46 -0400 Subject: [PATCH 15/50] Make applicationLayers @Nullable --- .../google/cloud/tools/jib/builder/steps/StepsRunner.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 8f18f760b9..b4872c5ac1 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -26,6 +26,7 @@ import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.image.Image; import com.google.common.base.Preconditions; +import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListeningExecutorService; @@ -60,7 +61,7 @@ private static Future failedFuture() { private Future baseImageAndAuth = failedFuture(); private Future>> baseImageLayers = failedFuture(); - private List> applicationLayers = new ArrayList<>(); + @Nullable private List> applicationLayers; private Future builtImage = failedFuture(); private Future targetRegistryCredentials = failedFuture(); private Future pushAuthorization = failedFuture(); @@ -176,7 +177,7 @@ private void buildImage() { childProgressDispatcherFactorySupplier.get(), results.baseImageAndAuth.get().getImage(), realizeFutures(results.baseImageLayers.get()), - realizeFutures(results.applicationLayers)) + realizeFutures(Verify.verifyNotNull(results.applicationLayers))) .call()); } @@ -201,7 +202,7 @@ private void pushApplicationLayers() { buildConfiguration, childProgressDispatcherFactorySupplier.get(), results.pushAuthorization.get(), - results.applicationLayers))); + Verify.verifyNotNull(results.applicationLayers)))); } private void pushImage() { From 0f6bc133194c14129eb47ae8f5b60c45a29b9d42 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 14 Jun 2019 15:03:58 -0400 Subject: [PATCH 16/50] Cleanup --- .../google/cloud/tools/jib/builder/steps/StepsRunner.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index f3d7a49df7..cd3f8eeee6 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -181,13 +181,6 @@ public BuildResult run() throws ExecutionException, InterruptedException { } } - private void enqueueBuildAndCache() { - stepsToRun.add(this::pullBaseImage); - stepsToRun.add(this::pullAndCacheBaseImageLayers); - stepsToRun.add(this::buildAndCacheApplicationLayers); - stepsToRun.add(this::buildImage); - } - private void retrieveTargetRegistryCredentials() { results.targetRegistryCredentials = executorService.submit( From 2bb59d8cbecdb7a65667e875cdf92588306a6da9 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 17 Jun 2019 18:27:40 -0400 Subject: [PATCH 17/50] Remove incorrect @VisibleForTesting --- .../jib/builder/steps/RetrieveRegistryCredentialsStep.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStep.java index c8ed60b9f7..15da9bc07f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/RetrieveRegistryCredentialsStep.java @@ -23,7 +23,6 @@ import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import java.util.Optional; import java.util.concurrent.Callable; @@ -60,7 +59,6 @@ static RetrieveRegistryCredentialsStep forTargetImage( private final String registry; private final ImmutableList credentialRetrievers; - @VisibleForTesting RetrieveRegistryCredentialsStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, From 6ee9ef07b925f5d49e1cd7b046dd487a669b3eb2 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 17 Jun 2019 18:42:12 -0400 Subject: [PATCH 18/50] Refactor code --- ...shImageStep.java => PushManifestStep.java} | 53 ++++++++++--------- .../tools/jib/builder/steps/StepsRunner.java | 2 +- 2 files changed, 28 insertions(+), 27 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/{PushImageStep.java => PushManifestStep.java} (72%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushManifestStep.java similarity index 72% rename from jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushManifestStep.java index 142b1e2014..c5ff30d5c5 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushManifestStep.java @@ -23,6 +23,7 @@ import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.event.EventHandlers; import com.google.cloud.tools.jib.hash.Digests; import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.image.Image; @@ -35,15 +36,14 @@ import java.util.concurrent.Callable; /** - * Pushes the final image. In fact, its manifest is pushed along with a tag. Returns the pushed - * image digest (digest of manifest) and image ID (digest of container configuration) as {#link - * BuildResult}. + * Pushes a manifest for a tag. Returns the manifest digest ("image digest") and the container + * configuration digest ("image id") as {#link BuildResult}. */ -class PushImageStep implements Callable { +class PushManifestStep implements Callable { - private static final String DESCRIPTION = "Pushing new image (tag)"; + private static final String DESCRIPTION = "Pushing manifest"; - static ImmutableList makeList( + static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Authorization pushAuthorization, @@ -54,9 +54,9 @@ static ImmutableList makeList( try (TimerEventDispatcher ignored = new TimerEventDispatcher( - buildConfiguration.getEventHandlers(), "Preparing image (tag) pushers"); + buildConfiguration.getEventHandlers(), "Preparing manifest pushers"); ProgressEventDispatcher progressEventDispatcher = - progressEventDispatcherFactory.create("preparing (image) tag pushers", tags.size())) { + progressEventDispatcherFactory.create("preparing manifest pushers", tags.size())) { // Gets the image manifest to push. BuildableManifestTemplate manifestTemplate = @@ -64,23 +64,24 @@ static ImmutableList makeList( .getManifestTemplate( buildConfiguration.getTargetFormat(), containerConfigurationDigestAndSize); - ImmutableList.Builder imagePushers = new ImmutableList.Builder<>(); - for (String tag : buildConfiguration.getAllTargetImageTags()) { - imagePushers.add( - new PushImageStep( - buildConfiguration, - progressEventDispatcher.newChildProducer(), - pushAuthorization, - manifestTemplate, - tag, - Digests.computeJsonDigest(manifestTemplate), - containerConfigurationDigestAndSize.getDigest())); - } - return imagePushers.build(); + DescriptorDigest manifestDigest = Digests.computeJsonDigest(manifestTemplate); + + return tags.stream() + .map( + tag -> + new PushManifestStep( + buildConfiguration, + progressEventDispatcher.newChildProducer(), + pushAuthorization, + manifestTemplate, + tag, + manifestDigest, + containerConfigurationDigestAndSize.getDigest())) + .collect(ImmutableList.toImmutableList()); } } - PushImageStep( + PushManifestStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Authorization pushAuthorization, @@ -108,11 +109,11 @@ static ImmutableList makeList( @Override public BuildResult call() throws IOException, RegistryException { - try (TimerEventDispatcher ignored = - new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION); + EventHandlers eventHandlers = buildConfiguration.getEventHandlers(); + try (TimerEventDispatcher ignored = new TimerEventDispatcher(eventHandlers, DESCRIPTION); ProgressEventDispatcher ignored2 = - progressEventDispatcherFactory.create("tagging with " + tag, 1)) { - buildConfiguration.getEventHandlers().dispatch(LogEvent.info("Tagging with " + tag + "...")); + progressEventDispatcherFactory.create("pushing manifest for " + tag, 1)) { + eventHandlers.dispatch(LogEvent.info("Pushing manifest for " + tag + "...")); RegistryClient registryClient = buildConfiguration diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index cd3f8eeee6..7c1e075c6d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -282,7 +282,7 @@ private void pushImages() { List> tagPushResults = scheduleCallables( - PushImageStep.makeList( + PushManifestStep.makeList( buildConfiguration, childProgressDispatcherFactorySupplier.get(), results.pushAuthorization.get(), From 157567693fe4eb0d84d7f2ebda244b1c555c496c Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 21 Jun 2019 13:31:42 -0400 Subject: [PATCH 19/50] Delete files --- .../PullAndCacheBaseImageLayersStep.java | 59 -------- .../jib/builder/steps/PushImageStep.java | 126 ------------------ 2 files changed, 185 deletions(-) delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java deleted file mode 100644 index 1ade0d839e..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayersStep.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.builder.steps; - -import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; -import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; -import com.google.cloud.tools.jib.configuration.BuildConfiguration; -import com.google.cloud.tools.jib.image.Layer; -import com.google.common.collect.ImmutableList; -import java.util.ArrayList; -import java.util.List; - -/** Pulls and caches the base image layers. */ -// TODO: following the same pattern as "BuildAndCacheApplicationLayerStep", move the sole "makeList" -// static into "PullAndCacheBaseImageLayerStep" and remove this class. -class PullAndCacheBaseImageLayersStep { - - private static final String DESCRIPTION = "Preparing base image layer pullers"; - - static ImmutableList makeList( - BuildConfiguration buildConfiguration, - ProgressEventDispatcher.Factory progressEventDispatcherFactory, - ImageAndAuthorization baseImageAndAuth) { - ImmutableList baseImageLayers = baseImageAndAuth.getImage().getLayers(); - - try (ProgressEventDispatcher progressEventDispatcher = - progressEventDispatcherFactory.create( - "preparing base image layer pullers", baseImageLayers.size()); - TimerEventDispatcher ignored = - new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { - - List layerPullers = new ArrayList<>(); - for (Layer layer : baseImageLayers) { - layerPullers.add( - new PullAndCacheBaseImageLayerStep( - buildConfiguration, - progressEventDispatcher.newChildProducer(), - layer.getBlobDescriptor().getDigest(), - baseImageAndAuth.getAuthorization())); - } - return ImmutableList.copyOf(layerPullers); - } - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java deleted file mode 100644 index 7cbb0172e4..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2018 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.builder.steps; - -import com.google.cloud.tools.jib.api.DescriptorDigest; -import com.google.cloud.tools.jib.api.LogEvent; -import com.google.cloud.tools.jib.blob.BlobDescriptor; -import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; -import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.configuration.BuildConfiguration; -import com.google.cloud.tools.jib.hash.Digests; -import com.google.cloud.tools.jib.http.Authorization; -import com.google.cloud.tools.jib.image.Image; -import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; -import com.google.cloud.tools.jib.image.json.ImageToJsonTranslator; -import com.google.cloud.tools.jib.registry.RegistryClient; -import com.google.common.collect.ImmutableSet; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; - -/** Pushes the final image. Outputs the pushed image digest. */ -class PushImageStep implements Callable { - - private static final String DESCRIPTION = "Pushing new image"; - - private final BuildConfiguration buildConfiguration; - private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - - private final Authorization pushAuthorization; - private final BlobDescriptor containerConfigurationDigestAndSize; - private final Image builtImage; - - private final ListeningExecutorService listeningExecutorService; - - // TODO: remove listeningExecutorService like other siblings - PushImageStep( - ListeningExecutorService listeningExecutorService, - BuildConfiguration buildConfiguration, - ProgressEventDispatcher.Factory progressEventDispatcherFactory, - Authorization pushAuthorization, - BlobDescriptor containerConfigurationDigestAndSize, - Image builtImage) { - this.listeningExecutorService = listeningExecutorService; - this.buildConfiguration = buildConfiguration; - this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.pushAuthorization = pushAuthorization; - this.containerConfigurationDigestAndSize = containerConfigurationDigestAndSize; - this.builtImage = builtImage; - } - - @Override - public BuildResult call() throws IOException, InterruptedException, ExecutionException { - ImmutableSet targetImageTags = buildConfiguration.getAllTargetImageTags(); - ProgressEventDispatcher progressEventDispatcher = - progressEventDispatcherFactory.create("pushing image manifest", targetImageTags.size()); - - try (TimerEventDispatcher ignored = - new TimerEventDispatcher(buildConfiguration.getEventHandlers(), DESCRIPTION)) { - RegistryClient registryClient = - buildConfiguration - .newTargetImageRegistryClientFactory() - .setAuthorization(pushAuthorization) - .newRegistryClient(); - - // Constructs the image. - ImageToJsonTranslator imageToJsonTranslator = new ImageToJsonTranslator(builtImage); - - // Gets the image manifest to push. - BuildableManifestTemplate manifestTemplate = - imageToJsonTranslator.getManifestTemplate( - buildConfiguration.getTargetFormat(), containerConfigurationDigestAndSize); - - // Pushes to all target image tags. - List> pushAllTagsFutures = new ArrayList<>(); - for (String tag : targetImageTags) { - ProgressEventDispatcher.Factory progressEventDispatcherFactory = - progressEventDispatcher.newChildProducer(); - pushAllTagsFutures.add( - listeningExecutorService.submit( - () -> { - try (ProgressEventDispatcher ignored2 = - progressEventDispatcherFactory.create("tagging with " + tag, 1)) { - buildConfiguration - .getEventHandlers() - .dispatch(LogEvent.info("Tagging with " + tag + "...")); - registryClient.pushManifest(manifestTemplate, tag); - } - return null; - })); - } - - DescriptorDigest imageDigest = Digests.computeJsonDigest(manifestTemplate); - DescriptorDigest imageId = containerConfigurationDigestAndSize.getDigest(); - BuildResult result = new BuildResult(imageDigest, imageId); - - return Futures.whenAllSucceed(pushAllTagsFutures) - .call( - () -> { - progressEventDispatcher.close(); - return result; - }, - listeningExecutorService) - .get(); - } - } -} From b1c3722fb3e1bc581ae9f4733a33d0ca9d96b901 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Fri, 21 Jun 2019 13:32:50 -0400 Subject: [PATCH 20/50] Revert renaming --- .../steps/{PushManifestStep.java => PushImageStep.java} | 8 ++++---- .../google/cloud/tools/jib/builder/steps/StepsRunner.java | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/{PushManifestStep.java => PushImageStep.java} (96%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushManifestStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java similarity index 96% rename from jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushManifestStep.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java index c5ff30d5c5..54fdae1893 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushManifestStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java @@ -39,11 +39,11 @@ * Pushes a manifest for a tag. Returns the manifest digest ("image digest") and the container * configuration digest ("image id") as {#link BuildResult}. */ -class PushManifestStep implements Callable { +class PushImageStep implements Callable { private static final String DESCRIPTION = "Pushing manifest"; - static ImmutableList makeList( + static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Authorization pushAuthorization, @@ -69,7 +69,7 @@ static ImmutableList makeList( return tags.stream() .map( tag -> - new PushManifestStep( + new PushImageStep( buildConfiguration, progressEventDispatcher.newChildProducer(), pushAuthorization, @@ -81,7 +81,7 @@ static ImmutableList makeList( } } - PushManifestStep( + PushImageStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Authorization pushAuthorization, diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 6772782c7d..a2a723e885 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -308,7 +308,7 @@ private void pushImages() { List> manifestPushResults = scheduleCallables( - PushManifestStep.makeList( + PushImageStep.makeList( buildConfiguration, childProgressDispatcherFactory, results.pushAuthorization.get(), From 5db41259f528d896dee47f6fbd7b51665a2d2494 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 24 Jun 2019 15:19:21 -0400 Subject: [PATCH 21/50] Fix Nullaway --- .../tools/jib/builder/steps/PushImageStep.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java index 6a3d60be9e..cdd0f4da7f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushImageStep.java @@ -82,6 +82,15 @@ static ImmutableList makeList( } } + private final BuildConfiguration buildConfiguration; + private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; + + private final BuildableManifestTemplate manifestTemplate; + @Nullable private final Authorization pushAuthorization; + private final String tag; + private final DescriptorDigest imageDigest; + private final DescriptorDigest imageId; + PushImageStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, @@ -99,15 +108,6 @@ static ImmutableList makeList( this.imageId = imageId; } - private final BuildConfiguration buildConfiguration; - private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - - private final BuildableManifestTemplate manifestTemplate; - private final Authorization pushAuthorization; - private final String tag; - private final DescriptorDigest imageDigest; - private final DescriptorDigest imageId; - @Override public BuildResult call() throws IOException, RegistryException { EventHandlers eventHandlers = buildConfiguration.getEventHandlers(); From 2889b0440078c67e1138e594a13d82f76b5a911b Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 26 Jun 2019 17:17:40 -0400 Subject: [PATCH 22/50] Prototyping (inefficient) --- jib-core/CHANGELOG.md | 2 + .../steps/PullAndCacheBaseImageLayerStep.java | 54 +++++++++++++++++++ .../tools/jib/builder/steps/StepsRunner.java | 48 ++++++++++++++--- .../cloud/tools/jib/cache/CachedLayer.java | 16 +++--- .../jib/image/json/ImageToJsonTranslator.java | 5 +- 5 files changed, 108 insertions(+), 17 deletions(-) diff --git a/jib-core/CHANGELOG.md b/jib-core/CHANGELOG.md index 9e336804b2..b677263a5a 100644 --- a/jib-core/CHANGELOG.md +++ b/jib-core/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. ### Added +- Say something + ### Changed - `JibContainerBuilder#addDependencies` is now split into three methods: `addDependencies`, `addSnapshotDependencies`, `addProjectDependencies` ([#1773](https://github.com/GoogleContainerTools/jib/pull/1773)) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 0868aef0ac..6bc8458628 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -17,6 +17,7 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.api.DescriptorDigest; +import com.google.cloud.tools.jib.api.RegistryException; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; @@ -33,6 +34,7 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.Callable; +import java.util.stream.Collectors; import javax.annotation.Nullable; /** Pulls and caches a single base image layer. */ @@ -40,6 +42,58 @@ class PullAndCacheBaseImageLayerStep implements Callable { private static final String DESCRIPTION = "Pulling base image layer %s"; + static boolean canSkipCachingBaseImageLayers( + BuildConfiguration buildConfiguration, + ImageAndAuthorization baseImageAndAuth, + @Nullable Authorization pushAuthorization) { + RegistryClient targetRegistryClient = + buildConfiguration + .newTargetImageRegistryClientFactory() + .setAuthorization(pushAuthorization) + .newRegistryClient(); + + try { + // TODO: parallelize + for (Layer layer : baseImageAndAuth.getImage().getLayers()) { + System.out.println(">>> Checking BLOb existence."); + System.out.println(">>> Size=" + layer.getBlobDescriptor().getSize()); + System.out.println(">>> Digest=" + layer.getBlobDescriptor().getDigest()); + System.out.println(">>> DiffId=" + layer.getDiffId()); + if (targetRegistryClient.checkBlob(layer.getBlobDescriptor().getDigest()) == null) { + System.out.println(">>> BLOb does not exist. Halt."); + return false; + } + } + System.out.println(">>> All BLObs exist. Can skip downloading base image layers."); + return true; + } catch (IOException | RegistryException ex) { + ex.printStackTrace(); + // fall through + } + System.out.println(">>> Exception occured."); + return false; + } + + static List pretendLayersCached(ImageAndAuthorization baseImageAndAuth) { + return baseImageAndAuth + .getImage() + .getLayers() + .stream() + .map( + layer -> + CachedLayer.builder() + .setLayerDigest(layer.getBlobDescriptor().getDigest()) + .setLayerSize(layer.getBlobDescriptor().getSize()) + .setLayerDiffId(layer.getDiffId()) + .setLayerBlob( + ignored -> { + throw new IllegalStateException("No actual BLOb attached"); + }) + .build()) + .map(cachedLayer -> new CachedLayerAndName(cachedLayer, null)) + .collect(Collectors.toList()); + } + static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index ec5ffe2ea6..97b6a29eda 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -69,6 +69,8 @@ private static Future failedFuture() { private Future>> applicationLayerPushResults = failedFuture(); private Future containerConfigurationPushResult = failedFuture(); private Future buildResult = failedFuture(); + + private Future canSkipCachingBaseImageLayers = Futures.immediateFuture(false); } /** @@ -146,6 +148,8 @@ public StepsRunner registryPushSteps() { rootProgressDescription = "building image to registry"; // build and cache stepsToRun.add(this::pullBaseImage); + stepsToRun.add(this::authenticatePush); + stepsToRun.add(this::checkCanSkipCachingBaseImageLayers); stepsToRun.add(this::pullAndCacheBaseImageLayers); stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); @@ -212,18 +216,40 @@ private void pullBaseImage() { new PullBaseImageStep(buildConfiguration, childProgressDispatcherFactory)); } + private void checkCanSkipCachingBaseImageLayers() { + Verify.verifyNotNull(rootProgressDispatcher).dispatchProgress(1); + + results.canSkipCachingBaseImageLayers = + executorService.submit( + () -> + PullAndCacheBaseImageLayerStep.canSkipCachingBaseImageLayers( + buildConfiguration, + results.baseImageAndAuth.get(), + results.pushAuthorization.get().orElse(null))); + } + private void pullAndCacheBaseImageLayers() { ProgressEventDispatcher.Factory childProgressDispatcherFactory = Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); results.baseImageLayers = executorService.submit( - () -> - scheduleCallables( + () -> { + if (results.canSkipCachingBaseImageLayers.get()) { + // optimization: skip downloading and caching base image layers + return PullAndCacheBaseImageLayerStep.pretendLayersCached( + results.baseImageAndAuth.get()) + .stream() + .map(Futures::immediateFuture) + .collect(Collectors.toList()); + } else { + return scheduleCallables( PullAndCacheBaseImageLayerStep.makeList( buildConfiguration, childProgressDispatcherFactory, - results.baseImageAndAuth.get()))); + results.baseImageAndAuth.get())); + } + }); } private void pushBaseImageLayers() { @@ -232,13 +258,23 @@ private void pushBaseImageLayers() { results.baseImageLayerPushResults = executorService.submit( - () -> - scheduleCallables( + () -> { + if (results.canSkipCachingBaseImageLayers.get()) { + // In this case, baseImageLayers are immediate futures (no blocking). + return realizeFutures(results.baseImageLayers.get()) + .stream() + .map(layer -> layer.getCachedLayer().getBlobDescriptor()) + .map(Futures::immediateFuture) + .collect(Collectors.toList()); + } else { + return scheduleCallables( PushLayerStep.makeList( buildConfiguration, childProgressDispatcherFactory, results.pushAuthorization.get().orElse(null), - results.baseImageLayers.get()))); + results.baseImageLayers.get())); + } + }); } private void buildAndCacheApplicationLayers() { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java index 43a543693a..3246bd21a2 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java @@ -27,7 +27,7 @@ public class CachedLayer implements Layer { /** Builds a {@link CachedLayer}. */ - static class Builder { + public static class Builder { @Nullable private DescriptorDigest layerDigest; @Nullable private DescriptorDigest layerDiffId; @@ -36,31 +36,31 @@ static class Builder { private Builder() {} - Builder setLayerDigest(DescriptorDigest layerDigest) { + public Builder setLayerDigest(DescriptorDigest layerDigest) { this.layerDigest = layerDigest; return this; } - Builder setLayerDiffId(DescriptorDigest layerDiffId) { + public Builder setLayerDiffId(DescriptorDigest layerDiffId) { this.layerDiffId = layerDiffId; return this; } - Builder setLayerSize(long layerSize) { + public Builder setLayerSize(long layerSize) { this.layerSize = layerSize; return this; } - Builder setLayerBlob(Blob layerBlob) { + public Builder setLayerBlob(Blob layerBlob) { this.layerBlob = layerBlob; return this; } - boolean hasLayerBlob() { + public boolean hasLayerBlob() { return layerBlob != null; } - CachedLayer build() { + public CachedLayer build() { return new CachedLayer( Preconditions.checkNotNull(layerDigest, "layerDigest required"), Preconditions.checkNotNull(layerDiffId, "layerDiffId required"), @@ -74,7 +74,7 @@ CachedLayer build() { * * @return the new {@link Builder} */ - static Builder builder() { + public static Builder builder() { return new Builder(); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java index 3f903b89f3..ddc8cf79d1 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.api.AbsoluteUnixPath; import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.api.Port; -import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.configuration.DockerHealthCheck; import com.google.cloud.tools.jib.image.Image; @@ -130,9 +129,9 @@ public ImageToJsonTranslator(Image image) { } /** - * Gets the container configuration as a {@link Blob}. + * Gets the container configuration. * - * @return the container configuration {@link Blob} + * @return the container configuration */ public JsonTemplate getContainerConfiguration() { // Set up the JSON template. From e8a5506751aad04bb890d44132e59082b366dbee Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 27 Jun 2019 11:53:14 -0400 Subject: [PATCH 23/50] wip --- .../jib/builder/steps/CheckBlobStep.java | 102 ++++++++++++++++++ .../steps/PullAndCacheBaseImageLayerStep.java | 22 +++- .../tools/jib/builder/steps/PushBlobStep.java | 10 +- .../tools/jib/builder/steps/StepsRunner.java | 66 +++++++----- .../image/LayerPropertyNotFoundException.java | 2 +- 5 files changed, 167 insertions(+), 35 deletions(-) create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CheckBlobStep.java diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CheckBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CheckBlobStep.java new file mode 100644 index 0000000000..6fe777b084 --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CheckBlobStep.java @@ -0,0 +1,102 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.builder.steps; + +import com.google.cloud.tools.jib.api.DescriptorDigest; +import com.google.cloud.tools.jib.api.RegistryException; +import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; +import com.google.cloud.tools.jib.builder.TimerEventDispatcher; +import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.http.Authorization; +import com.google.cloud.tools.jib.image.Image; +import com.google.cloud.tools.jib.registry.RegistryClient; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.util.concurrent.Callable; +import javax.annotation.Nullable; + +/** Checks if a BLOb exists in a registry. */ +class CheckBlobStep implements Callable { + + static ImmutableList makeList( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, + Image baseImage, + @Nullable Authorization pushAuthorization) { + try (TimerEventDispatcher ignored = + new TimerEventDispatcher( + buildConfiguration.getEventHandlers(), "Preparing blob checkers "); + ProgressEventDispatcher progressEventDispatcher = + progressEventDispatcherFactory.create( + "preparing blob checkers", baseImage.getLayers().size())) { + return baseImage + .getLayers() + .stream() + .map( + layer -> + new CheckBlobStep( + buildConfiguration, + progressEventDispatcher.newChildProducer(), + pushAuthorization, + layer.getBlobDescriptor().getDigest(), + true)) + .collect(ImmutableList.toImmutableList()); + } + } + + private final BuildConfiguration buildConfiguration; + private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; + @Nullable private final Authorization pushAuthorization; + private final DescriptorDigest blobDigest; + private final boolean ignoreError; + + CheckBlobStep( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, + @Nullable Authorization pushAuthorization, + DescriptorDigest blobDigest, + boolean ignoreError) { + this.buildConfiguration = buildConfiguration; + this.progressEventDispatcherFactory = progressEventDispatcherFactory; + this.pushAuthorization = pushAuthorization; + this.blobDigest = blobDigest; + this.ignoreError = ignoreError; + } + + @Override + public Boolean call() throws Exception { + try (ProgressEventDispatcher progressEventDispatcher = + progressEventDispatcherFactory.create("checking blob " + blobDigest, 1); + TimerEventDispatcher ignored = + new TimerEventDispatcher( + buildConfiguration.getEventHandlers(), "Checking blob " + blobDigest)) { + + RegistryClient registryClient = + buildConfiguration + .newTargetImageRegistryClientFactory() + .setAuthorization(pushAuthorization) + .newRegistryClient(); + return registryClient.checkBlob(blobDigest) != null; + + } catch (IOException | RegistryException ex) { + if (ignoreError) { + return false; + } + throw ex; + } + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 6bc8458628..1dae3c4f8d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -26,7 +26,9 @@ import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.http.Authorization; +import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; +import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; import com.google.cloud.tools.jib.registry.RegistryClient; import com.google.common.collect.ImmutableList; import java.io.IOException; @@ -74,9 +76,19 @@ static boolean canSkipCachingBaseImageLayers( return false; } - static List pretendLayersCached(ImageAndAuthorization baseImageAndAuth) { - return baseImageAndAuth - .getImage() + static List createNoBlobCachedLayers( + BuildConfiguration buildConfiguration, Image baseImage) + throws IOException, CacheCorruptedException { + // The image manifest is already saved, so we should delete it if not all of the layers are + // actually cached. (--offline shouldn't see an incomplete caching state.) + Cache cache = buildConfiguration.getBaseImageLayersCache(); + for (Layer layer : baseImage.getLayers()) { + if (!cache.retrieve(layer.getBlobDescriptor().getDigest()).isPresent()) { + // TODO: delete the manifest. + } + } + + return baseImage .getLayers() .stream() .map( @@ -87,7 +99,7 @@ static List pretendLayersCached(ImageAndAuthorization baseIm .setLayerDiffId(layer.getDiffId()) .setLayerBlob( ignored -> { - throw new IllegalStateException("No actual BLOb attached"); + throw new LayerPropertyNotFoundException("No actual BLOb attached"); }) .build()) .map(cachedLayer -> new CachedLayerAndName(cachedLayer, null)) @@ -154,7 +166,7 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException { throw new IOException( "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " + layerDigest - + ". You may need to rerun Jib in online mode to re-download the base image layers."); + + ". Rerun Jib in online mode to re-download the base image layers."); } RegistryClient registryClient = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java index 19f4eafa6b..56a41f966d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java @@ -36,7 +36,7 @@ class PushBlobStep implements Callable { private static final String DESCRIPTION = "Pushing BLOB "; private final BuildConfiguration buildConfiguration; - private final ProgressEventDispatcher.Factory progressEventDipatcherFactory; + private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; @Nullable private final Authorization authorization; private final BlobDescriptor blobDescriptor; @@ -44,12 +44,12 @@ class PushBlobStep implements Callable { PushBlobStep( BuildConfiguration buildConfiguration, - ProgressEventDispatcher.Factory progressEventDipatcherFactory, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, @Nullable Authorization authorization, BlobDescriptor blobDescriptor, Blob blob) { this.buildConfiguration = buildConfiguration; - this.progressEventDipatcherFactory = progressEventDipatcherFactory; + this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.authorization = authorization; this.blobDescriptor = blobDescriptor; this.blob = blob; @@ -58,7 +58,7 @@ class PushBlobStep implements Callable { @Override public BlobDescriptor call() throws IOException, RegistryException { try (ProgressEventDispatcher progressEventDispatcher = - progressEventDipatcherFactory.create( + progressEventDispatcherFactory.create( "pushing blob " + blobDescriptor.getDigest(), blobDescriptor.getSize()); TimerEventDispatcher ignored = new TimerEventDispatcher( @@ -71,6 +71,8 @@ public BlobDescriptor call() throws IOException, RegistryException { .setAuthorization(authorization) .newRegistryClient(); + // TODO: for base image layers, we may skip the check, since we did the check from + // CheckBlobStep.makeList(). // check if the BLOB is available if (registryClient.checkBlob(blobDescriptor.getDigest()) != null) { buildConfiguration diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 97b6a29eda..d6f77cc162 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -33,6 +33,7 @@ import com.google.common.util.concurrent.MoreExecutors; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.Callable; @@ -70,6 +71,7 @@ private static Future failedFuture() { private Future containerConfigurationPushResult = failedFuture(); private Future buildResult = failedFuture(); + private Future>> baseImageLayersExistInTargetRegistry = failedFuture(); private Future canSkipCachingBaseImageLayers = Futures.immediateFuture(false); } @@ -146,16 +148,16 @@ public StepsRunner tarBuildSteps(Path outputPath) { public StepsRunner registryPushSteps() { rootProgressDescription = "building image to registry"; + + stepsToRun.add(this::retrieveTargetRegistryCredentials); + stepsToRun.add(this::authenticatePush); // build and cache stepsToRun.add(this::pullBaseImage); - stepsToRun.add(this::authenticatePush); - stepsToRun.add(this::checkCanSkipCachingBaseImageLayers); + stepsToRun.add(this::checkBaseImageLayersInTargetRegistry); stepsToRun.add(this::pullAndCacheBaseImageLayers); stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); // push to registry - stepsToRun.add(this::retrieveTargetRegistryCredentials); - stepsToRun.add(this::authenticatePush); stepsToRun.add(this::pushBaseImageLayers); stepsToRun.add(this::pushApplicationLayers); stepsToRun.add(this::pushContainerConfiguration); @@ -216,6 +218,21 @@ private void pullBaseImage() { new PullBaseImageStep(buildConfiguration, childProgressDispatcherFactory)); } + private void checkBaseImageLayersInTargetRegistry() { + ProgressEventDispatcher.Factory childProgressDispatcherFactory = + Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); + + results.baseImageLayersExistInTargetRegistry = + executorService.submit( + () -> + scheduleCallables( + CheckBlobStep.makeList( + buildConfiguration, + childProgressDispatcherFactory, + results.baseImageAndAuth.get().getImage(), + results.pushAuthorization.get().orElse(null)))); + } + private void checkCanSkipCachingBaseImageLayers() { Verify.verifyNotNull(rootProgressDispatcher).dispatchProgress(1); @@ -228,6 +245,12 @@ private void checkCanSkipCachingBaseImageLayers() { results.pushAuthorization.get().orElse(null))); } + private boolean canSkipPushingBaseImageLayers() throws InterruptedException, ExecutionException { + // TODO: we can optimize to short-curcuit as soon as any future evaluates to false. + List layersExist = realizeFutures(results.baseImageLayersExistInTargetRegistry.get()); + return layersExist.stream().allMatch(Boolean.TRUE::equals); + } + private void pullAndCacheBaseImageLayers() { ProgressEventDispatcher.Factory childProgressDispatcherFactory = Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); @@ -235,10 +258,11 @@ private void pullAndCacheBaseImageLayers() { results.baseImageLayers = executorService.submit( () -> { - if (results.canSkipCachingBaseImageLayers.get()) { - // optimization: skip downloading and caching base image layers - return PullAndCacheBaseImageLayerStep.pretendLayersCached( - results.baseImageAndAuth.get()) + if (canSkipPushingBaseImageLayers()) { + System.out.println(">>> Can skip pushing base image layers"); + // optimization: skip downloading/caching base image layers + return PullAndCacheBaseImageLayerStep.createNoBlobCachedLayers( + buildConfiguration, results.baseImageAndAuth.get().getImage()) .stream() .map(Futures::immediateFuture) .collect(Collectors.toList()); @@ -258,23 +282,15 @@ private void pushBaseImageLayers() { results.baseImageLayerPushResults = executorService.submit( - () -> { - if (results.canSkipCachingBaseImageLayers.get()) { - // In this case, baseImageLayers are immediate futures (no blocking). - return realizeFutures(results.baseImageLayers.get()) - .stream() - .map(layer -> layer.getCachedLayer().getBlobDescriptor()) - .map(Futures::immediateFuture) - .collect(Collectors.toList()); - } else { - return scheduleCallables( - PushLayerStep.makeList( - buildConfiguration, - childProgressDispatcherFactory, - results.pushAuthorization.get().orElse(null), - results.baseImageLayers.get())); - } - }); + () -> + canSkipPushingBaseImageLayers() + ? Collections.emptyList() + : scheduleCallables( + PushLayerStep.makeList( + buildConfiguration, + childProgressDispatcherFactory, + results.pushAuthorization.get().orElse(null), + results.baseImageLayers.get()))); } private void buildAndCacheApplicationLayers() { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerPropertyNotFoundException.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerPropertyNotFoundException.java index 3424bbd741..75524e0038 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerPropertyNotFoundException.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerPropertyNotFoundException.java @@ -19,7 +19,7 @@ /** Exception thrown when accessing non-existent properties of layers. */ public class LayerPropertyNotFoundException extends RuntimeException { - LayerPropertyNotFoundException(String message) { + public LayerPropertyNotFoundException(String message) { super(message); } } From ea165f8978ee0a962941446976aca78aebd50900 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 27 Jun 2019 12:00:20 -0400 Subject: [PATCH 24/50] Minor fixes --- .../jib/image/json/ImageToJsonTranslator.java | 18 +++--------------- .../json/V22ManifestListTemplateTest.java | 3 +-- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java index 407c333348..ed2ac78181 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ImageToJsonTranslator.java @@ -19,7 +19,6 @@ import com.google.cloud.tools.jib.api.AbsoluteUnixPath; import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.api.Port; -import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.configuration.DockerHealthCheck; import com.google.cloud.tools.jib.image.Image; @@ -36,18 +35,7 @@ import java.util.function.Function; import javax.annotation.Nullable; -/** - * Translates an {@link Image} into a manifest or container configuration JSON BLOB. - * - *

Example usage: - * - * 

{@code
- * ImageToJsonTranslator translator = new ImageToJsonTranslator(image);
- * Blob containerConfigurationBlob = translator.getContainerConfigurationBlob();
- * BlobDescriptor containerConfigurationBlobDescriptor = blob.writeTo(outputStream);
- * Blob manifestBlob = translator.getManifestBlob(containerConfigurationBlobDescriptor);
- * }
- */ +/** Translates an {@link Image} into a manifest or container configuration JSON. */ public class ImageToJsonTranslator { /** @@ -141,9 +129,9 @@ public ImageToJsonTranslator(Image image) { } /** - * Gets the container configuration as a {@link Blob}. + * Gets the container configuration. * - * @return the container configuration {@link Blob} + * @return the container configuration */ public JsonTemplate getContainerConfiguration() { // Set up the JSON template. diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplateTest.java index 0db48e9cc3..87002549f8 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplateTest.java @@ -23,7 +23,6 @@ import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.DigestException; import java.util.List; import org.junit.Assert; import org.junit.Test; @@ -31,7 +30,7 @@ public class V22ManifestListTemplateTest { @Test - public void testFromJson() throws IOException, URISyntaxException, DigestException { + public void testFromJson() throws IOException, URISyntaxException { Path jsonFile = Paths.get(Resources.getResource("core/json/v22manifest_list.json").toURI()); V22ManifestListTemplate manifestListJson = From f78954a99f463b9129793e95ffd9d8e42295a89f Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 27 Jun 2019 18:10:10 -0400 Subject: [PATCH 25/50] wip --- .../BuildAndCacheApplicationLayerStep.java | 34 ++++++++--------- .../steps/PullAndCacheBaseImageLayerStep.java | 38 +++++++++++++++---- .../tools/jib/builder/steps/PushBlobStep.java | 19 +++++----- .../steps/PushContainerConfigurationStep.java | 3 +- .../jib/builder/steps/PushLayerStep.java | 32 +++++++++------- .../tools/jib/builder/steps/StepsRunner.java | 30 ++++++++++----- 6 files changed, 97 insertions(+), 59 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java index 632720f85a..324f0010b0 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java @@ -29,6 +29,7 @@ import com.google.cloud.tools.jib.image.ReproducibleLayerBuilder; import com.google.common.collect.ImmutableList; import java.io.IOException; +import java.util.List; import java.util.Optional; import java.util.concurrent.Callable; @@ -44,30 +45,27 @@ class BuildAndCacheApplicationLayerStep implements Callable static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory) { - int layerCount = buildConfiguration.getLayerConfigurations().size(); + List layerConfigurations = buildConfiguration.getLayerConfigurations(); try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create( - "preparing application layer builders", layerCount); + "preparing application layer builders", layerConfigurations.size()); TimerEventDispatcher ignored = new TimerEventDispatcher( buildConfiguration.getEventHandlers(), "Preparing application layer builders")) { - ImmutableList.Builder buildAndCacheApplicationLayerSteps = - ImmutableList.builderWithExpectedSize(layerCount); - for (LayerConfiguration layerConfiguration : buildConfiguration.getLayerConfigurations()) { - // Skips the layer if empty. - if (layerConfiguration.getLayerEntries().isEmpty()) { - continue; - } - - buildAndCacheApplicationLayerSteps.add( - new BuildAndCacheApplicationLayerStep( - buildConfiguration, - progressEventDispatcher.newChildProducer(), - layerConfiguration.getName(), - layerConfiguration)); - } - return buildAndCacheApplicationLayerSteps.build(); + + return layerConfigurations + .stream() + // Skips the layer if empty. + .filter(layerConfiguration -> !layerConfiguration.getLayerEntries().isEmpty()) + .map( + layerConfiguration -> + new BuildAndCacheApplicationLayerStep( + buildConfiguration, + progressEventDispatcher.newChildProducer(), + layerConfiguration.getName(), + layerConfiguration)) + .collect(ImmutableList.toImmutableList()); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 1dae3c4f8d..9f267b9d58 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -109,7 +109,9 @@ static List createNoBlobCachedLayers( static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, - ImageAndAuthorization baseImageAndAuth) { + ImageAndAuthorization baseImageAndAuth, + boolean registryPush, + Authorization pushAuthorization) { ImmutableList baseImageLayers = baseImageAndAuth.getImage().getLayers(); try (ProgressEventDispatcher progressEventDispatcher = @@ -125,8 +127,10 @@ static ImmutableList makeList( new PullAndCacheBaseImageLayerStep( buildConfiguration, progressEventDispatcher.newChildProducer(), - layer.getBlobDescriptor().getDigest(), - baseImageAndAuth.getAuthorization())); + layer, + baseImageAndAuth.getAuthorization(), + pushAuthorization, + registryPush)); } return ImmutableList.copyOf(layerPullers); } @@ -135,22 +139,29 @@ static ImmutableList makeList( private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - private final DescriptorDigest layerDigest; + private final Layer layer; private final @Nullable Authorization pullAuthorization; + private final @Nullable Authorization pushAuthorization; + private final boolean registryPush; PullAndCacheBaseImageLayerStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, - DescriptorDigest layerDigest, - @Nullable Authorization pullAuthorization) { + Layer layer, + @Nullable Authorization pullAuthorization, + @Nullable Authorization pushAuthorization, + boolean registryPush) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.layerDigest = layerDigest; + this.layer = layer; this.pullAuthorization = pullAuthorization; + this.pushAuthorization = pushAuthorization; + this.registryPush = registryPush; } @Override - public CachedLayerAndName call() throws IOException, CacheCorruptedException { + public CachedLayerAndName call() throws IOException, CacheCorruptedException, RegistryException { + DescriptorDigest layerDigest = layer.getBlobDescriptor().getDigest(); try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create("checking base image layer " + layerDigest, 1); TimerEventDispatcher ignored = @@ -169,6 +180,17 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException { + ". Rerun Jib in online mode to re-download the base image layers."); } + if (registryPush) { + RegistryClient targetRegistryClient = + buildConfiguration + .newTargetImageRegistryClientFactory() + .setAuthorization(pullAuthorization) + .newRegistryClient(); + if (targetRegistryClient.checkBlob(layerDigest) != null) { + return new CachedLayerAndName(layer, null); + } + } + RegistryClient registryClient = buildConfiguration .newBaseImageRegistryClientFactory() diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java index 56a41f966d..9784d66fb9 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java @@ -23,6 +23,7 @@ import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.event.EventHandlers; import com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer; import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.registry.RegistryClient; @@ -41,28 +42,31 @@ class PushBlobStep implements Callable { @Nullable private final Authorization authorization; private final BlobDescriptor blobDescriptor; private final Blob blob; + private final boolean doBlobCheck; PushBlobStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, @Nullable Authorization authorization, BlobDescriptor blobDescriptor, - Blob blob) { + Blob blob, + boolean doBlobCheck) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.authorization = authorization; this.blobDescriptor = blobDescriptor; this.blob = blob; + this.doBlobCheck = doBlobCheck; } @Override public BlobDescriptor call() throws IOException, RegistryException { + EventHandlers eventHandlers = buildConfiguration.getEventHandlers(); try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create( "pushing blob " + blobDescriptor.getDigest(), blobDescriptor.getSize()); TimerEventDispatcher ignored = - new TimerEventDispatcher( - buildConfiguration.getEventHandlers(), DESCRIPTION + blobDescriptor); + new TimerEventDispatcher(eventHandlers, DESCRIPTION + blobDescriptor); ThrottledAccumulatingConsumer throttledProgressReporter = new ThrottledAccumulatingConsumer(progressEventDispatcher::dispatchProgress)) { RegistryClient registryClient = @@ -71,13 +75,10 @@ public BlobDescriptor call() throws IOException, RegistryException { .setAuthorization(authorization) .newRegistryClient(); - // TODO: for base image layers, we may skip the check, since we did the check from - // CheckBlobStep.makeList(). // check if the BLOB is available - if (registryClient.checkBlob(blobDescriptor.getDigest()) != null) { - buildConfiguration - .getEventHandlers() - .dispatch(LogEvent.info("BLOB : " + blobDescriptor + " already exists on registry")); + if (doBlobCheck && registryClient.checkBlob(blobDescriptor.getDigest()) != null) { + eventHandlers.dispatch( + LogEvent.info("BLOB : " + blobDescriptor + " already exists on registry")); return blobDescriptor; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java index 87e12d064b..966b39574d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java @@ -67,7 +67,8 @@ public BlobDescriptor call() throws IOException, RegistryException { progressEventDispatcher.newChildProducer(), pushAuthorization, Digests.computeDigest(containerConfiguration), - Blobs.from(containerConfiguration)) + Blobs.from(containerConfiguration), + true) .call(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java index 05fb34b8f9..a1c8f4b379 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java @@ -25,7 +25,6 @@ import com.google.cloud.tools.jib.http.Authorization; import com.google.common.collect.ImmutableList; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -38,7 +37,8 @@ static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, @Nullable Authorization pushAuthorization, - List> cachedLayers) { + List> cachedLayers, + boolean doBlobCheck) { try (TimerEventDispatcher ignored = new TimerEventDispatcher( buildConfiguration.getEventHandlers(), "Preparing application layer pushers"); @@ -47,15 +47,17 @@ static ImmutableList makeList( "preparing application layer pushers", cachedLayers.size())) { // Constructs a PushBlobStep for each layer. - List blobPushers = new ArrayList<>(); - for (Future layer : cachedLayers) { - ProgressEventDispatcher.Factory childProgressProducer = - progressEventDispatcher.newChildProducer(); - blobPushers.add( - new PushLayerStep(buildConfiguration, childProgressProducer, pushAuthorization, layer)); - } - - return ImmutableList.copyOf(blobPushers); + return cachedLayers + .stream() + .map( + layer -> + new PushLayerStep( + buildConfiguration, + progressEventDispatcher.newChildProducer(), + pushAuthorization, + layer, + doBlobCheck)) + .collect(ImmutableList.toImmutableList()); } } @@ -64,16 +66,19 @@ static ImmutableList makeList( @Nullable private final Authorization pushAuthorization; private final Future cachedLayerAndName; + private final boolean doBlobCheck; PushLayerStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, @Nullable Authorization pushAuthorization, - Future cachedLayerAndName) { + Future cachedLayerAndName, + boolean doBlobCheck) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.pushAuthorization = pushAuthorization; this.cachedLayerAndName = cachedLayerAndName; + this.doBlobCheck = doBlobCheck; } @Override @@ -85,7 +90,8 @@ public BlobDescriptor call() progressEventDispatcherFactory, pushAuthorization, new BlobDescriptor(layer.getSize(), layer.getDigest()), - layer.getBlob()) + layer.getBlob(), + doBlobCheck) .call(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index d6f77cc162..95744c7a8d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -33,7 +33,6 @@ import com.google.common.util.concurrent.MoreExecutors; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.Callable; @@ -104,6 +103,8 @@ private static List realizeFutures(List> futures) private final ExecutorService executorService; private final BuildConfiguration buildConfiguration; + private boolean registryPush; + // We save steps to run by wrapping each step into a Runnable, only because of the unfortunate // chicken-and-egg situation arising from using ProgressEventDispatcher. The current // ProgressEventDispatcher model requires knowing in advance how many units of work (i.e., steps) @@ -124,6 +125,8 @@ private StepsRunner( public StepsRunner dockerLoadSteps(DockerClient dockerClient) { rootProgressDescription = "building image to Docker daemon"; + registryPush = false; + // build and cache stepsToRun.add(this::pullBaseImage); stepsToRun.add(this::pullAndCacheBaseImageLayers); @@ -136,6 +139,8 @@ public StepsRunner dockerLoadSteps(DockerClient dockerClient) { public StepsRunner tarBuildSteps(Path outputPath) { rootProgressDescription = "building image to tar file"; + registryPush = false; + // build and cache stepsToRun.add(this::pullBaseImage); stepsToRun.add(this::pullAndCacheBaseImageLayers); @@ -148,6 +153,7 @@ public StepsRunner tarBuildSteps(Path outputPath) { public StepsRunner registryPushSteps() { rootProgressDescription = "building image to registry"; + registryPush = true; stepsToRun.add(this::retrieveTargetRegistryCredentials); stepsToRun.add(this::authenticatePush); @@ -258,6 +264,9 @@ private void pullAndCacheBaseImageLayers() { results.baseImageLayers = executorService.submit( () -> { + Authorization pushAuthorization = + registryPush ? results.pushAuthorization.get().orElse(null) : null; + if (canSkipPushingBaseImageLayers()) { System.out.println(">>> Can skip pushing base image layers"); // optimization: skip downloading/caching base image layers @@ -271,7 +280,9 @@ private void pullAndCacheBaseImageLayers() { PullAndCacheBaseImageLayerStep.makeList( buildConfiguration, childProgressDispatcherFactory, - results.baseImageAndAuth.get())); + results.baseImageAndAuth.get(), + registryPush, + pushAuthorization)); } }); } @@ -283,14 +294,13 @@ private void pushBaseImageLayers() { results.baseImageLayerPushResults = executorService.submit( () -> - canSkipPushingBaseImageLayers() - ? Collections.emptyList() - : scheduleCallables( - PushLayerStep.makeList( - buildConfiguration, - childProgressDispatcherFactory, - results.pushAuthorization.get().orElse(null), - results.baseImageLayers.get()))); + scheduleCallables( + PushLayerStep.makeList( + buildConfiguration, + childProgressDispatcherFactory, + results.pushAuthorization.get().orElse(null), + results.baseImageLayers.get(), + registryPush))); } private void buildAndCacheApplicationLayers() { From b723d5420e1e9568bac9fda224086d15feccd5cc Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 10 Jul 2019 14:28:22 -0400 Subject: [PATCH 26/50] Introduce PreparedLayer --- .../BuildAndCacheApplicationLayerStep.java | 10 +- .../jib/builder/steps/BuildImageStep.java | 19 +-- .../jib/builder/steps/CachedLayerAndName.java | 41 ----- .../jib/builder/steps/PreparedLayer.java | 103 ++++++++++++ .../steps/PullAndCacheBaseImageLayerStep.java | 149 +++++++++++------ .../jib/builder/steps/PushLayerStep.java | 28 ++-- .../tools/jib/builder/steps/StepsRunner.java | 158 +++++++++--------- ...BuildAndCacheApplicationLayerStepTest.java | 2 +- .../jib/builder/steps/BuildImageStepTest.java | 18 +- 9 files changed, 323 insertions(+), 205 deletions(-) delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java index 96a3d3cafd..a802d5d81f 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java @@ -34,7 +34,7 @@ import java.util.concurrent.Callable; /** Builds and caches application layers. */ -class BuildAndCacheApplicationLayerStep implements Callable { +class BuildAndCacheApplicationLayerStep implements Callable { private static final String DESCRIPTION = "Building %s layer"; @@ -86,7 +86,7 @@ private BuildAndCacheApplicationLayerStep( } @Override - public CachedLayerAndName call() throws IOException, CacheCorruptedException { + public PreparedLayer call() throws IOException, CacheCorruptedException { String description = String.format(DESCRIPTION, layerName); EventHandlers eventHandlers = buildConfiguration.getEventHandlers(); @@ -95,13 +95,15 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException { try (ProgressEventDispatcher ignored = progressEventDispatcherFactory.create("building " + layerName + " layer", 1); TimerEventDispatcher ignored2 = new TimerEventDispatcher(eventHandlers, description)) { + // TODO: for registry push, check blob (layer) in the target registry to skip building layer. + Cache cache = buildConfiguration.getApplicationLayersCache(); // Don't build the layer if it exists already. Optional optionalCachedLayer = cache.retrieve(layerConfiguration.getLayerEntries()); if (optionalCachedLayer.isPresent()) { - return new CachedLayerAndName(optionalCachedLayer.get(), layerName); + return new PreparedLayer.Builder(optionalCachedLayer.get()).setName(layerName).build(); } Blob layerBlob = new ReproducibleLayerBuilder(layerConfiguration.getLayerEntries()).build(); @@ -110,7 +112,7 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException { eventHandlers.dispatch(LogEvent.debug(description + " built " + cachedLayer.getDigest())); - return new CachedLayerAndName(cachedLayer, layerName); + return new PreparedLayer.Builder(cachedLayer).setName(layerName).build(); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java index bb3783eb75..e221317557 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java @@ -25,7 +25,6 @@ import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; import com.google.cloud.tools.jib.image.json.HistoryEntry; -import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import java.time.Instant; import java.util.List; @@ -40,15 +39,15 @@ class BuildImageStep implements Callable { private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; private final Image baseImage; - private final List baseImageLayers; - private final List applicationLayers; + private final List baseImageLayers; + private final List applicationLayers; BuildImageStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Image baseImage, - List baseImageLayers, - List applicationLayers) { + List baseImageLayers, + List applicationLayers) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.baseImage = baseImage; @@ -68,9 +67,7 @@ public Image call() throws LayerPropertyNotFoundException { buildConfiguration.getContainerConfiguration(); // Base image layers - for (CachedLayerAndName baseImageLayer : baseImageLayers) { - imageBuilder.addLayer(baseImageLayer.getCachedLayer()); - } + baseImageLayers.stream().forEach(imageBuilder::addLayer); // Passthrough config and count non-empty history entries int nonEmptyLayerCount = 0; @@ -104,15 +101,15 @@ public Image call() throws LayerPropertyNotFoundException { } // Add built layers/configuration - for (CachedLayerAndName applicationLayer : applicationLayers) { + for (PreparedLayer applicationLayer : applicationLayers) { imageBuilder - .addLayer(applicationLayer.getCachedLayer()) + .addLayer(applicationLayer) .addHistory( HistoryEntry.builder() .setCreationTimestamp(layerCreationTime) .setAuthor("Jib") .setCreatedBy(buildConfiguration.getToolName() + ":" + ProjectInfo.VERSION) - .setComment(Verify.verifyNotNull(applicationLayer.getName())) + .setComment(applicationLayer.getName()) .build()); } if (containerConfiguration != null) { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java deleted file mode 100644 index eed59fc14e..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2019 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.builder.steps; - -import com.google.cloud.tools.jib.cache.CachedLayer; -import javax.annotation.Nullable; - -/** Simple structure to hold a pair of {#link CachedLayer} and its string name. */ -class CachedLayerAndName { - - private CachedLayer cachedLayer; - @Nullable private String name; - - CachedLayerAndName(CachedLayer cachedLayer, @Nullable String name) { - this.cachedLayer = cachedLayer; - this.name = name; - } - - CachedLayer getCachedLayer() { - return cachedLayer; - } - - @Nullable - String getName() { - return name; - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java new file mode 100644 index 0000000000..942a7be1c6 --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java @@ -0,0 +1,103 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.builder.steps; + +import com.google.cloud.tools.jib.api.DescriptorDigest; +import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.image.Layer; + +/** + * Layer prepared from {@link BuildAndCacheApplicationLayerStep} and {@link + * PullAndCacheBaseImageLayerStep} to hold information about either a base image layer or an + * application layer. + */ +class PreparedLayer implements Layer { + + /** + * Represents if this layer is known to exist or not in the target destination (for example, + * registry, local Docker daemon). + */ + static enum StateInTarget { + UNKNOWN, + EXISTS, + DOES_NOT_EXIST + }; + + static class Builder { + + private Layer layer; + private String name = "unnamed layer"; + private StateInTarget stateInTarget = StateInTarget.UNKNOWN; + + Builder(Layer layer) { + this.layer = layer; + } + + Builder setName(String name) { + this.name = name; + return this; + } + + Builder existsInTarget() { + this.stateInTarget = StateInTarget.EXISTS; + return this; + } + + public Builder doesNotExistInTarget() { + this.stateInTarget = StateInTarget.DOES_NOT_EXIST; + return this; + } + + PreparedLayer build() { + return new PreparedLayer(layer, stateInTarget, name); + } + } + + private final Layer layer; + private final StateInTarget stateInTarget; + private final String name; + + private PreparedLayer(Layer layer, StateInTarget stateInTarget, String name) { + this.layer = layer; + this.stateInTarget = stateInTarget; + this.name = name; + } + + String getName() { + return name; + } + + StateInTarget getStateInTarget() { + return stateInTarget; + } + + @Override + public Blob getBlob() { + return layer.getBlob(); + } + + @Override + public BlobDescriptor getBlobDescriptor() { + return layer.getBlobDescriptor(); + } + + @Override + public DescriptorDigest getDiffId() { + return layer.getDiffId(); + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 9f267b9d58..d39a3b9908 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -26,25 +26,25 @@ import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.http.Authorization; -import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; import com.google.cloud.tools.jib.registry.RegistryClient; +import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.Callable; -import java.util.stream.Collectors; +import java.util.function.Predicate; import javax.annotation.Nullable; /** Pulls and caches a single base image layer. */ -class PullAndCacheBaseImageLayerStep implements Callable { +class PullAndCacheBaseImageLayerStep implements Callable { private static final String DESCRIPTION = "Pulling base image layer %s"; - static boolean canSkipCachingBaseImageLayers( + private static boolean canSkipCachingBaseImageLayers( BuildConfiguration buildConfiguration, ImageAndAuthorization baseImageAndAuth, @Nullable Authorization pushAuthorization) { @@ -76,37 +76,78 @@ static boolean canSkipCachingBaseImageLayers( return false; } - static List createNoBlobCachedLayers( - BuildConfiguration buildConfiguration, Image baseImage) - throws IOException, CacheCorruptedException { - // The image manifest is already saved, so we should delete it if not all of the layers are - // actually cached. (--offline shouldn't see an incomplete caching state.) - Cache cache = buildConfiguration.getBaseImageLayersCache(); - for (Layer layer : baseImage.getLayers()) { - if (!cache.retrieve(layer.getBlobDescriptor().getDigest()).isPresent()) { - // TODO: delete the manifest. - } - } + // private static List createNoBlobCachedLayers( + // BuildConfiguration buildConfiguration, Image baseImage) + // throws IOException, CacheCorruptedException { + // // The image manifest is already saved, so we should delete it if not all of the layers are + // // actually cached. (--offline shouldn't see an incomplete caching state.) + // Cache cache = buildConfiguration.getBaseImageLayersCache(); + // for (Layer layer : baseImage.getLayers()) { + // if (!cache.retrieve(layer.getBlobDescriptor().getDigest()).isPresent()) { + // // TODO: delete the manifest. + // } + // } + // + // return baseImage + // .getLayers() + // .stream() + // .map( + // layer -> + // CachedLayer.builder() + // .setLayerDigest(layer.getBlobDescriptor().getDigest()) + // .setLayerSize(layer.getBlobDescriptor().getSize()) + // .setLayerDiffId(layer.getDiffId()) + // .setLayerBlob( + // ignored -> { + // throw new LayerPropertyNotFoundException("No actual BLOb attached"); + // }) + // .build()) + // .map(cachedLayer -> new PreparedLayer(cachedLayer, true, null)) + // .collect(Collectors.toList()); + // } - return baseImage - .getLayers() - .stream() - .map( - layer -> - CachedLayer.builder() - .setLayerDigest(layer.getBlobDescriptor().getDigest()) - .setLayerSize(layer.getBlobDescriptor().getSize()) - .setLayerDiffId(layer.getDiffId()) - .setLayerBlob( - ignored -> { - throw new LayerPropertyNotFoundException("No actual BLOb attached"); - }) - .build()) - .map(cachedLayer -> new CachedLayerAndName(cachedLayer, null)) - .collect(Collectors.toList()); + static ImmutableList makeListForcedDownload( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, + ImageAndAuthorization baseImageAndAuth) { + return makeList( + buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, false, null); } - static ImmutableList makeList( + static ImmutableList makeListOptimized( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, + ImageAndAuthorization baseImageAndAuth, + Authorization pushAuthorization) { + Predicate blobChecker = + layer -> { + buildConfiguration + .newTargetImageRegistryClientFactory() + .setAuthorization(pushAuthorization) + .newRegistryClient(); + return true; + }; + blobChecker.test(null); + return makeList( + buildConfiguration, + progressEventDispatcherFactory, + baseImageAndAuth, + true, + pushAuthorization); + } + + private static boolean layerExistsInTargetRegistry( + BuildConfiguration buildConfiguration, Authorization authorization, Layer layer) + throws LayerPropertyNotFoundException, IOException, RegistryException { + RegistryClient registryClient = + buildConfiguration + .newTargetImageRegistryClientFactory() + .setAuthorization(authorization) + .newRegistryClient(); + return registryClient.checkBlob(layer.getBlobDescriptor().getDigest()) != null; + } + + private static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth, @@ -121,6 +162,16 @@ static ImmutableList makeList( new TimerEventDispatcher( buildConfiguration.getEventHandlers(), "Preparing base image layer pullers")) { + boolean checkBlob = registryPush && !Boolean.getBoolean("jib.forceDownload"); + if (checkBlob) { + Verify.verify(!buildConfiguration.isOffline()); + + buildConfiguration + .newTargetImageRegistryClientFactory() + .setAuthorization(pushAuthorization) + .newRegistryClient(); + } + List layerPullers = new ArrayList<>(); for (Layer layer : baseImageLayers) { layerPullers.add( @@ -160,7 +211,7 @@ static ImmutableList makeList( } @Override - public CachedLayerAndName call() throws IOException, CacheCorruptedException, RegistryException { + public PreparedLayer call() throws IOException, CacheCorruptedException, RegistryException { DescriptorDigest layerDigest = layer.getBlobDescriptor().getDigest(); try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create("checking base image layer " + layerDigest, 1); @@ -168,29 +219,33 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException, Re new TimerEventDispatcher( buildConfiguration.getEventHandlers(), String.format(DESCRIPTION, layerDigest))) { Cache cache = buildConfiguration.getBaseImageLayersCache(); - - // Checks if the layer already exists in the cache. Optional optionalCachedLayer = cache.retrieve(layerDigest); - if (optionalCachedLayer.isPresent()) { - return new CachedLayerAndName(optionalCachedLayer.get(), null); - } else if (buildConfiguration.isOffline()) { - throw new IOException( - "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " - + layerDigest - + ". Rerun Jib in online mode to re-download the base image layers."); - } - if (registryPush) { + boolean checkBlob = registryPush && !Boolean.getBoolean("jib.forceDownload"); + if (checkBlob) { + Verify.verify(!buildConfiguration.isOffline()); + RegistryClient targetRegistryClient = buildConfiguration .newTargetImageRegistryClientFactory() - .setAuthorization(pullAuthorization) + .setAuthorization(pushAuthorization) .newRegistryClient(); if (targetRegistryClient.checkBlob(layerDigest) != null) { - return new CachedLayerAndName(layer, null); + return new PreparedLayer.Builder(layer).existsInTarget().build(); } } + // Checks if the layer already exists in the cache. + if (optionalCachedLayer.isPresent()) { + return new PreparedLayer.Builder(optionalCachedLayer.get()).doesNotExistInTarget().build(); + } else if (buildConfiguration.isOffline()) { + throw new IOException( + "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " + + layerDigest + + ". Rerun Jib in online mode with \"-Djib.forceDownload=true\" to re-download the " + + "base image layers."); + } + RegistryClient registryClient = buildConfiguration .newBaseImageRegistryClientFactory() @@ -207,7 +262,7 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException, Re layerDigest, progressEventDispatcherWrapper::setProgressTarget, progressEventDispatcherWrapper::dispatchProgress)); - return new CachedLayerAndName(cachedLayer, null); + return new PreparedLayer.Builder(cachedLayer).build(); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java index a1c8f4b379..b1bb5ef6cd 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java @@ -20,7 +20,7 @@ import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.cache.CachedLayer; +import com.google.cloud.tools.jib.builder.steps.PreparedLayer.StateInTarget; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.http.Authorization; import com.google.common.collect.ImmutableList; @@ -37,8 +37,7 @@ static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, @Nullable Authorization pushAuthorization, - List> cachedLayers, - boolean doBlobCheck) { + List> cachedLayers) { try (TimerEventDispatcher ignored = new TimerEventDispatcher( buildConfiguration.getEventHandlers(), "Preparing application layer pushers"); @@ -55,8 +54,7 @@ static ImmutableList makeList( buildConfiguration, progressEventDispatcher.newChildProducer(), pushAuthorization, - layer, - doBlobCheck)) + layer)) .collect(ImmutableList.toImmutableList()); } } @@ -65,33 +63,35 @@ static ImmutableList makeList( private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; @Nullable private final Authorization pushAuthorization; - private final Future cachedLayerAndName; - private final boolean doBlobCheck; + private final Future preparedLayer; PushLayerStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, @Nullable Authorization pushAuthorization, - Future cachedLayerAndName, - boolean doBlobCheck) { + Future preparedLayer) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.pushAuthorization = pushAuthorization; - this.cachedLayerAndName = cachedLayerAndName; - this.doBlobCheck = doBlobCheck; + this.preparedLayer = preparedLayer; } @Override public BlobDescriptor call() throws IOException, RegistryException, ExecutionException, InterruptedException { - CachedLayer layer = cachedLayerAndName.get().getCachedLayer(); + PreparedLayer layer = preparedLayer.get(); + if (layer.getStateInTarget() == StateInTarget.EXISTS) { + return layer.getBlobDescriptor(); + } + + boolean checkBlob = layer.getStateInTarget() == StateInTarget.UNKNOWN; return new PushBlobStep( buildConfiguration, progressEventDispatcherFactory, pushAuthorization, - new BlobDescriptor(layer.getSize(), layer.getDigest()), + layer.getBlobDescriptor(), layer.getBlob(), - doBlobCheck) + checkBlob) .call(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 2ab8a7e1f6..da88829e9a 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -60,8 +60,8 @@ private static Future failedFuture() { } private Future baseImageAndAuth = failedFuture(); - private Future>> baseImageLayers = failedFuture(); - @Nullable private List> applicationLayers; + private Future>> baseImageLayers = failedFuture(); + @Nullable private List> applicationLayers; private Future builtImage = failedFuture(); private Future> targetRegistryCredentials = failedFuture(); private Future> pushAuthorization = failedFuture(); @@ -69,9 +69,10 @@ private static Future failedFuture() { private Future>> applicationLayerPushResults = failedFuture(); private Future containerConfigurationPushResult = failedFuture(); private Future buildResult = failedFuture(); - - private Future>> baseImageLayersExistInTargetRegistry = failedFuture(); - private Future canSkipCachingBaseImageLayers = Futures.immediateFuture(false); + // + // private Future>> baseImageLayersExistInTargetRegistry = + // failedFuture(); + // private Future canSkipCachingBaseImageLayers = Futures.immediateFuture(false); } /** @@ -103,8 +104,6 @@ private static List realizeFutures(List> futures) private final ExecutorService executorService; private final BuildConfiguration buildConfiguration; - private boolean registryPush; - // We save steps to run by wrapping each step into a Runnable, only because of the unfortunate // chicken-and-egg situation arising from using ProgressEventDispatcher. The current // ProgressEventDispatcher model requires knowing in advance how many units of work (i.e., steps) @@ -125,11 +124,10 @@ private StepsRunner( public StepsRunner dockerLoadSteps(DockerClient dockerClient) { rootProgressDescription = "building image to Docker daemon"; - registryPush = false; // build and cache stepsToRun.add(this::pullBaseImage); - stepsToRun.add(this::pullAndCacheBaseImageLayers); + stepsToRun.add(() -> pullAndCacheBaseImageLayers(true)); stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); // load to Docker @@ -139,11 +137,10 @@ public StepsRunner dockerLoadSteps(DockerClient dockerClient) { public StepsRunner tarBuildSteps(Path outputPath) { rootProgressDescription = "building image to tar file"; - registryPush = false; // build and cache stepsToRun.add(this::pullBaseImage); - stepsToRun.add(this::pullAndCacheBaseImageLayers); + stepsToRun.add(() -> pullAndCacheBaseImageLayers(true)); stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); // create a tar @@ -153,17 +150,16 @@ public StepsRunner tarBuildSteps(Path outputPath) { public StepsRunner registryPushSteps() { rootProgressDescription = "building image to registry"; - registryPush = true; stepsToRun.add(this::retrieveTargetRegistryCredentials); stepsToRun.add(this::authenticatePush); - // build and cache + stepsToRun.add(this::pullBaseImage); - stepsToRun.add(this::checkBaseImageLayersInTargetRegistry); - stepsToRun.add(this::pullAndCacheBaseImageLayers); + stepsToRun.add(() -> pullAndCacheBaseImageLayers(Boolean.getBoolean("jib.forceDownload"))); stepsToRun.add(this::buildAndCacheApplicationLayers); + // stepsToRun.add(this::checkBaseImageLayersInTargetRegistry); stepsToRun.add(this::buildImage); - // push to registry + stepsToRun.add(this::pushBaseImageLayers); stepsToRun.add(this::pushApplicationLayers); stepsToRun.add(this::pushContainerConfiguration); @@ -224,67 +220,75 @@ private void pullBaseImage() { new PullBaseImageStep(buildConfiguration, childProgressDispatcherFactory)); } - private void checkBaseImageLayersInTargetRegistry() { + // private void checkBaseImageLayersInTargetRegistry() { + // ProgressEventDispatcher.Factory childProgressDispatcherFactory = + // Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); + // + // results.baseImageLayersExistInTargetRegistry = + // executorService.submit( + // () -> + // scheduleCallables( + // CheckBlobStep.makeList( + // buildConfiguration, + // childProgressDispatcherFactory, + // results.baseImageAndAuth.get().getImage(), + // results.pushAuthorization.get().orElse(null)))); + // } + + // private void checkCanSkipCachingBaseImageLayers() { + // Verify.verifyNotNull(rootProgressDispatcher).dispatchProgress(1); + // + // results.canSkipCachingBaseImageLayers = + // executorService.submit( + // () -> + // PullAndCacheBaseImageLayerStep.canSkipCachingBaseImageLayers( + // buildConfiguration, + // results.baseImageAndAuth.get(), + // results.pushAuthorization.get().orElse(null))); + // } + + // private boolean canSkipPushingBaseImageLayers() throws InterruptedException, + // ExecutionException { + // // TODO: we can optimize to short-curcuit as soon as any future evaluates to false. + // List layersExist = + // realizeFutures(results.baseImageLayersExistInTargetRegistry.get()); + // return layersExist.stream().allMatch(Boolean.TRUE::equals); + // } + + private void pullAndCacheBaseImageLayers(boolean forcePull) { ProgressEventDispatcher.Factory childProgressDispatcherFactory = Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); - results.baseImageLayersExistInTargetRegistry = - executorService.submit( - () -> - scheduleCallables( - CheckBlobStep.makeList( - buildConfiguration, - childProgressDispatcherFactory, - results.baseImageAndAuth.get().getImage(), - results.pushAuthorization.get().orElse(null)))); - } - - private void checkCanSkipCachingBaseImageLayers() { - Verify.verifyNotNull(rootProgressDispatcher).dispatchProgress(1); - - results.canSkipCachingBaseImageLayers = - executorService.submit( - () -> - PullAndCacheBaseImageLayerStep.canSkipCachingBaseImageLayers( - buildConfiguration, - results.baseImageAndAuth.get(), - results.pushAuthorization.get().orElse(null))); - } - - private boolean canSkipPushingBaseImageLayers() throws InterruptedException, ExecutionException { - // TODO: we can optimize to short-curcuit as soon as any future evaluates to false. - List layersExist = realizeFutures(results.baseImageLayersExistInTargetRegistry.get()); - return layersExist.stream().allMatch(Boolean.TRUE::equals); - } - - private void pullAndCacheBaseImageLayers() { - ProgressEventDispatcher.Factory childProgressDispatcherFactory = - Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); - - results.baseImageLayers = - executorService.submit( - () -> { - Authorization pushAuthorization = - registryPush ? results.pushAuthorization.get().orElse(null) : null; - - if (canSkipPushingBaseImageLayers()) { - System.out.println(">>> Can skip pushing base image layers"); - // optimization: skip downloading/caching base image layers - return PullAndCacheBaseImageLayerStep.createNoBlobCachedLayers( - buildConfiguration, results.baseImageAndAuth.get().getImage()) - .stream() - .map(Futures::immediateFuture) - .collect(Collectors.toList()); - } else { - return scheduleCallables( - PullAndCacheBaseImageLayerStep.makeList( - buildConfiguration, - childProgressDispatcherFactory, - results.baseImageAndAuth.get(), - registryPush, - pushAuthorization)); - } - }); + if (forcePull) { + results.baseImageLayers = + executorService.submit( + () -> + scheduleCallables( + PullAndCacheBaseImageLayerStep.makeListForcedDownload( + buildConfiguration, + childProgressDispatcherFactory, + results.baseImageAndAuth.get()))); + } else { + results.baseImageLayers = + executorService.submit( + () -> + // if (canSkipPushingBaseImageLayers()) { + // System.out.println(">>> Can skip pushing base image layers"); + // // optimization: skip downloading/caching base image layers + // return PullAndCacheBaseImageLayerStep.createNoBlobCachedLayers( + // buildConfiguration, + // results.baseImageAndAuth.get().getImage()) + // .stream() + // .map(Futures::immediateFuture) + // .collect(Collectors.toList()); + // } else { + scheduleCallables( + PullAndCacheBaseImageLayerStep.makeListOptimized( + buildConfiguration, + childProgressDispatcherFactory, + results.baseImageAndAuth.get(), + results.pushAuthorization.get().orElse(null)))); + } } private void pushBaseImageLayers() { @@ -299,8 +303,7 @@ private void pushBaseImageLayers() { buildConfiguration, childProgressDispatcherFactory, results.pushAuthorization.get().orElse(null), - results.baseImageLayers.get(), - registryPush))); + results.baseImageLayers.get()))); } private void buildAndCacheApplicationLayers() { @@ -356,8 +359,7 @@ private void pushApplicationLayers() { buildConfiguration, childProgressDispatcherFactory, results.pushAuthorization.get().orElse(null), - Verify.verifyNotNull(results.applicationLayers), - true))); + Verify.verifyNotNull(results.applicationLayers)))); } private void pushImages() { diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java index 38a45bbf07..cc7d3c9bf9 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java @@ -135,7 +135,7 @@ private List buildFakeLayersToCache() for (BuildAndCacheApplicationLayerStep buildAndCacheApplicationLayerStep : buildAndCacheApplicationLayerSteps) { - applicationLayers.add(buildAndCacheApplicationLayerStep.call().getCachedLayer()); + applicationLayers.add(buildAndCacheApplicationLayerStep.call()); } return applicationLayers; diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java index 5fe9044450..c0d7040c5e 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java @@ -55,8 +55,8 @@ public class BuildImageStepTest { @Mock private CachedLayer mockCachedLayer; private Image baseImage; - private List baseImageLayers; - private List applicationLayers; + private List baseImageLayers; + private List applicationLayers; private DescriptorDigest testDescriptorDigest; private HistoryEntry nonEmptyLayerHistory; @@ -122,15 +122,15 @@ public void setUp() throws DigestException { .build(); baseImageLayers = Arrays.asList( - new CachedLayerAndName(mockCachedLayer, null), - new CachedLayerAndName(mockCachedLayer, null), - new CachedLayerAndName(mockCachedLayer, null)); + new PreparedLayer.Builder(mockCachedLayer).build(), + new PreparedLayer.Builder(mockCachedLayer).build(), + new PreparedLayer.Builder(mockCachedLayer).build()); applicationLayers = Arrays.asList( - new CachedLayerAndName(mockCachedLayer, "dependencies"), - new CachedLayerAndName(mockCachedLayer, "resources"), - new CachedLayerAndName(mockCachedLayer, "classes"), - new CachedLayerAndName(mockCachedLayer, "extra files")); + new PreparedLayer.Builder(mockCachedLayer).setName("dependencies").build(), + new PreparedLayer.Builder(mockCachedLayer).setName("resources").build(), + new PreparedLayer.Builder(mockCachedLayer).setName("classes").build(), + new PreparedLayer.Builder(mockCachedLayer).setName("extra files").build()); } @Test From e0e75e0048763717a253fee3d16288c5a80bd37d Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 10 Jul 2019 17:45:31 -0400 Subject: [PATCH 27/50] Refactor code --- .../jib/builder/steps/PreparedLayer.java | 31 ++---- .../steps/PullAndCacheBaseImageLayerStep.java | 95 +++++++------------ .../jib/builder/steps/PushLayerStep.java | 8 +- .../tools/jib/builder/steps/StepsRunner.java | 45 +++------ .../cloud/tools/jib/cache/CachedLayer.java | 16 ++-- 5 files changed, 70 insertions(+), 125 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java index 942a7be1c6..38e048ab1e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java @@ -20,6 +20,7 @@ import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.image.Layer; +import java.util.Optional; /** * Layer prepared from {@link BuildAndCacheApplicationLayerStep} and {@link @@ -28,21 +29,11 @@ */ class PreparedLayer implements Layer { - /** - * Represents if this layer is known to exist or not in the target destination (for example, - * registry, local Docker daemon). - */ - static enum StateInTarget { - UNKNOWN, - EXISTS, - DOES_NOT_EXIST - }; - static class Builder { private Layer layer; private String name = "unnamed layer"; - private StateInTarget stateInTarget = StateInTarget.UNKNOWN; + private Optional stateInTarget = Optional.empty(); Builder(Layer layer) { this.layer = layer; @@ -53,26 +44,22 @@ Builder setName(String name) { return this; } - Builder existsInTarget() { - this.stateInTarget = StateInTarget.EXISTS; - return this; - } - - public Builder doesNotExistInTarget() { - this.stateInTarget = StateInTarget.DOES_NOT_EXIST; + /** Sets whether the layer exists in a target destination. Empty means unknown. */ + Builder setStateInTarget(Optional stateInTarget) { + this.stateInTarget = stateInTarget; return this; } PreparedLayer build() { - return new PreparedLayer(layer, stateInTarget, name); + return new PreparedLayer(layer, name, stateInTarget); } } private final Layer layer; - private final StateInTarget stateInTarget; private final String name; + private final Optional stateInTarget; - private PreparedLayer(Layer layer, StateInTarget stateInTarget, String name) { + private PreparedLayer(Layer layer, String name, Optional stateInTarget) { this.layer = layer; this.stateInTarget = stateInTarget; this.name = name; @@ -82,7 +69,7 @@ String getName() { return name; } - StateInTarget getStateInTarget() { + Optional existsInTarget() { return stateInTarget; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index d39a3b9908..2eae3e6fc7 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -27,7 +27,6 @@ import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.image.Layer; -import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; import com.google.cloud.tools.jib.registry.RegistryClient; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; @@ -36,12 +35,16 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.Callable; -import java.util.function.Predicate; import javax.annotation.Nullable; /** Pulls and caches a single base image layer. */ class PullAndCacheBaseImageLayerStep implements Callable { + private interface LayerChecker { + + Optional exists(Layer layer) throws IOException, RegistryException; + } + private static final String DESCRIPTION = "Pulling base image layer %s"; private static boolean canSkipCachingBaseImageLayers( @@ -111,7 +114,10 @@ static ImmutableList makeListForcedDownload( ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth) { return makeList( - buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, false, null); + buildConfiguration, + progressEventDispatcherFactory, + baseImageAndAuth, + ignored -> Optional.empty()); } static ImmutableList makeListOptimized( @@ -119,40 +125,28 @@ static ImmutableList makeListOptimized( ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth, Authorization pushAuthorization) { - Predicate blobChecker = - layer -> { - buildConfiguration - .newTargetImageRegistryClientFactory() - .setAuthorization(pushAuthorization) - .newRegistryClient(); - return true; - }; - blobChecker.test(null); - return makeList( - buildConfiguration, - progressEventDispatcherFactory, - baseImageAndAuth, - true, - pushAuthorization); - } + Verify.verify(!buildConfiguration.isOffline()); - private static boolean layerExistsInTargetRegistry( - BuildConfiguration buildConfiguration, Authorization authorization, Layer layer) - throws LayerPropertyNotFoundException, IOException, RegistryException { - RegistryClient registryClient = + RegistryClient targetRegistryClient = buildConfiguration .newTargetImageRegistryClientFactory() - .setAuthorization(authorization) + .setAuthorization(pushAuthorization) .newRegistryClient(); - return registryClient.checkBlob(layer.getBlobDescriptor().getDigest()) != null; + + LayerChecker layerChecker = + layer -> + Optional.of( + targetRegistryClient.checkBlob(layer.getBlobDescriptor().getDigest()).isPresent()); + + return makeList( + buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, layerChecker); } private static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth, - boolean registryPush, - Authorization pushAuthorization) { + LayerChecker layerChecker) { ImmutableList baseImageLayers = baseImageAndAuth.getImage().getLayers(); try (ProgressEventDispatcher progressEventDispatcher = @@ -162,16 +156,6 @@ private static ImmutableList makeList( new TimerEventDispatcher( buildConfiguration.getEventHandlers(), "Preparing base image layer pullers")) { - boolean checkBlob = registryPush && !Boolean.getBoolean("jib.forceDownload"); - if (checkBlob) { - Verify.verify(!buildConfiguration.isOffline()); - - buildConfiguration - .newTargetImageRegistryClientFactory() - .setAuthorization(pushAuthorization) - .newRegistryClient(); - } - List layerPullers = new ArrayList<>(); for (Layer layer : baseImageLayers) { layerPullers.add( @@ -180,8 +164,7 @@ private static ImmutableList makeList( progressEventDispatcher.newChildProducer(), layer, baseImageAndAuth.getAuthorization(), - pushAuthorization, - registryPush)); + layerChecker)); } return ImmutableList.copyOf(layerPullers); } @@ -192,22 +175,19 @@ private static ImmutableList makeList( private final Layer layer; private final @Nullable Authorization pullAuthorization; - private final @Nullable Authorization pushAuthorization; - private final boolean registryPush; + private final LayerChecker layerChecker; PullAndCacheBaseImageLayerStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Layer layer, @Nullable Authorization pullAuthorization, - @Nullable Authorization pushAuthorization, - boolean registryPush) { + LayerChecker layerChecker) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.layer = layer; this.pullAuthorization = pullAuthorization; - this.pushAuthorization = pushAuthorization; - this.registryPush = registryPush; + this.layerChecker = layerChecker; } @Override @@ -218,26 +198,19 @@ public PreparedLayer call() throws IOException, CacheCorruptedException, Registr TimerEventDispatcher ignored = new TimerEventDispatcher( buildConfiguration.getEventHandlers(), String.format(DESCRIPTION, layerDigest))) { - Cache cache = buildConfiguration.getBaseImageLayersCache(); - Optional optionalCachedLayer = cache.retrieve(layerDigest); - - boolean checkBlob = registryPush && !Boolean.getBoolean("jib.forceDownload"); - if (checkBlob) { - Verify.verify(!buildConfiguration.isOffline()); - RegistryClient targetRegistryClient = - buildConfiguration - .newTargetImageRegistryClientFactory() - .setAuthorization(pushAuthorization) - .newRegistryClient(); - if (targetRegistryClient.checkBlob(layerDigest) != null) { - return new PreparedLayer.Builder(layer).existsInTarget().build(); - } + Optional layerExists = layerChecker.exists(layer); + if (layerExists.orElse(false)) { + return new PreparedLayer.Builder(layer).setStateInTarget(layerExists).build(); } + Cache cache = buildConfiguration.getBaseImageLayersCache(); + Optional optionalCachedLayer = cache.retrieve(layerDigest); + // Checks if the layer already exists in the cache. if (optionalCachedLayer.isPresent()) { - return new PreparedLayer.Builder(optionalCachedLayer.get()).doesNotExistInTarget().build(); + CachedLayer cachedLayer = optionalCachedLayer.get(); + return new PreparedLayer.Builder(cachedLayer).setStateInTarget(layerExists).build(); } else if (buildConfiguration.isOffline()) { throw new IOException( "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " @@ -262,7 +235,7 @@ public PreparedLayer call() throws IOException, CacheCorruptedException, Registr layerDigest, progressEventDispatcherWrapper::setProgressTarget, progressEventDispatcherWrapper::dispatchProgress)); - return new PreparedLayer.Builder(cachedLayer).build(); + return new PreparedLayer.Builder(cachedLayer).setStateInTarget(layerExists).build(); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java index b1bb5ef6cd..493a38ac9c 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java @@ -20,7 +20,6 @@ import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.builder.steps.PreparedLayer.StateInTarget; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.http.Authorization; import com.google.common.collect.ImmutableList; @@ -80,18 +79,19 @@ static ImmutableList makeList( public BlobDescriptor call() throws IOException, RegistryException, ExecutionException, InterruptedException { PreparedLayer layer = preparedLayer.get(); - if (layer.getStateInTarget() == StateInTarget.EXISTS) { + if (layer.existsInTarget().orElse(false)) { // skip pushing if known to exist in registry return layer.getBlobDescriptor(); } - boolean checkBlob = layer.getStateInTarget() == StateInTarget.UNKNOWN; + // Boolean object not present --> unknown whether or not the layer exists in registry + boolean doBlobCheck = !layer.existsInTarget().isPresent(); return new PushBlobStep( buildConfiguration, progressEventDispatcherFactory, pushAuthorization, layer.getBlobDescriptor(), layer.getBlob(), - checkBlob) + doBlobCheck) .call(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index da88829e9a..3bf4c70301 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -160,6 +160,7 @@ public StepsRunner registryPushSteps() { // stepsToRun.add(this::checkBaseImageLayersInTargetRegistry); stepsToRun.add(this::buildImage); + // push to registry stepsToRun.add(this::pushBaseImageLayers); stepsToRun.add(this::pushApplicationLayers); stepsToRun.add(this::pushContainerConfiguration); @@ -259,36 +260,20 @@ private void pullAndCacheBaseImageLayers(boolean forcePull) { ProgressEventDispatcher.Factory childProgressDispatcherFactory = Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); - if (forcePull) { - results.baseImageLayers = - executorService.submit( - () -> - scheduleCallables( - PullAndCacheBaseImageLayerStep.makeListForcedDownload( - buildConfiguration, - childProgressDispatcherFactory, - results.baseImageAndAuth.get()))); - } else { - results.baseImageLayers = - executorService.submit( - () -> - // if (canSkipPushingBaseImageLayers()) { - // System.out.println(">>> Can skip pushing base image layers"); - // // optimization: skip downloading/caching base image layers - // return PullAndCacheBaseImageLayerStep.createNoBlobCachedLayers( - // buildConfiguration, - // results.baseImageAndAuth.get().getImage()) - // .stream() - // .map(Futures::immediateFuture) - // .collect(Collectors.toList()); - // } else { - scheduleCallables( - PullAndCacheBaseImageLayerStep.makeListOptimized( - buildConfiguration, - childProgressDispatcherFactory, - results.baseImageAndAuth.get(), - results.pushAuthorization.get().orElse(null)))); - } + results.baseImageLayers = + executorService.submit( + () -> + scheduleCallables( + forcePull + ? PullAndCacheBaseImageLayerStep.makeListForcedDownload( + buildConfiguration, + childProgressDispatcherFactory, + results.baseImageAndAuth.get()) + : PullAndCacheBaseImageLayerStep.makeListOptimized( + buildConfiguration, + childProgressDispatcherFactory, + results.baseImageAndAuth.get(), + results.pushAuthorization.get().orElse(null)))); } private void pushBaseImageLayers() { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java index 3246bd21a2..43a543693a 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java @@ -27,7 +27,7 @@ public class CachedLayer implements Layer { /** Builds a {@link CachedLayer}. */ - public static class Builder { + static class Builder { @Nullable private DescriptorDigest layerDigest; @Nullable private DescriptorDigest layerDiffId; @@ -36,31 +36,31 @@ public static class Builder { private Builder() {} - public Builder setLayerDigest(DescriptorDigest layerDigest) { + Builder setLayerDigest(DescriptorDigest layerDigest) { this.layerDigest = layerDigest; return this; } - public Builder setLayerDiffId(DescriptorDigest layerDiffId) { + Builder setLayerDiffId(DescriptorDigest layerDiffId) { this.layerDiffId = layerDiffId; return this; } - public Builder setLayerSize(long layerSize) { + Builder setLayerSize(long layerSize) { this.layerSize = layerSize; return this; } - public Builder setLayerBlob(Blob layerBlob) { + Builder setLayerBlob(Blob layerBlob) { this.layerBlob = layerBlob; return this; } - public boolean hasLayerBlob() { + boolean hasLayerBlob() { return layerBlob != null; } - public CachedLayer build() { + CachedLayer build() { return new CachedLayer( Preconditions.checkNotNull(layerDigest, "layerDigest required"), Preconditions.checkNotNull(layerDiffId, "layerDiffId required"), @@ -74,7 +74,7 @@ public CachedLayer build() { * * @return the new {@link Builder} */ - public static Builder builder() { + static Builder builder() { return new Builder(); } From ed706067aaf7e977685e702f0fea2b4c5419e355 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 11 Jul 2019 10:47:39 -0400 Subject: [PATCH 28/50] wip --- .../jib/builder/steps/CheckBlobStep.java | 102 ------------------ .../jib/builder/steps/PreparedLayer.java | 2 +- .../steps/PullAndCacheBaseImageLayerStep.java | 37 +++---- .../tools/jib/builder/steps/PushBlobStep.java | 9 +- .../tools/jib/builder/steps/StepsRunner.java | 4 +- .../maven/BuildImageMojoIntegrationTest.java | 19 ++-- 6 files changed, 35 insertions(+), 138 deletions(-) delete mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CheckBlobStep.java diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CheckBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CheckBlobStep.java deleted file mode 100644 index 6fe777b084..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CheckBlobStep.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2019 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.builder.steps; - -import com.google.cloud.tools.jib.api.DescriptorDigest; -import com.google.cloud.tools.jib.api.RegistryException; -import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; -import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.configuration.BuildConfiguration; -import com.google.cloud.tools.jib.http.Authorization; -import com.google.cloud.tools.jib.image.Image; -import com.google.cloud.tools.jib.registry.RegistryClient; -import com.google.common.collect.ImmutableList; -import java.io.IOException; -import java.util.concurrent.Callable; -import javax.annotation.Nullable; - -/** Checks if a BLOb exists in a registry. */ -class CheckBlobStep implements Callable { - - static ImmutableList makeList( - BuildConfiguration buildConfiguration, - ProgressEventDispatcher.Factory progressEventDispatcherFactory, - Image baseImage, - @Nullable Authorization pushAuthorization) { - try (TimerEventDispatcher ignored = - new TimerEventDispatcher( - buildConfiguration.getEventHandlers(), "Preparing blob checkers "); - ProgressEventDispatcher progressEventDispatcher = - progressEventDispatcherFactory.create( - "preparing blob checkers", baseImage.getLayers().size())) { - return baseImage - .getLayers() - .stream() - .map( - layer -> - new CheckBlobStep( - buildConfiguration, - progressEventDispatcher.newChildProducer(), - pushAuthorization, - layer.getBlobDescriptor().getDigest(), - true)) - .collect(ImmutableList.toImmutableList()); - } - } - - private final BuildConfiguration buildConfiguration; - private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - @Nullable private final Authorization pushAuthorization; - private final DescriptorDigest blobDigest; - private final boolean ignoreError; - - CheckBlobStep( - BuildConfiguration buildConfiguration, - ProgressEventDispatcher.Factory progressEventDispatcherFactory, - @Nullable Authorization pushAuthorization, - DescriptorDigest blobDigest, - boolean ignoreError) { - this.buildConfiguration = buildConfiguration; - this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.pushAuthorization = pushAuthorization; - this.blobDigest = blobDigest; - this.ignoreError = ignoreError; - } - - @Override - public Boolean call() throws Exception { - try (ProgressEventDispatcher progressEventDispatcher = - progressEventDispatcherFactory.create("checking blob " + blobDigest, 1); - TimerEventDispatcher ignored = - new TimerEventDispatcher( - buildConfiguration.getEventHandlers(), "Checking blob " + blobDigest)) { - - RegistryClient registryClient = - buildConfiguration - .newTargetImageRegistryClientFactory() - .setAuthorization(pushAuthorization) - .newRegistryClient(); - return registryClient.checkBlob(blobDigest) != null; - - } catch (IOException | RegistryException ex) { - if (ignoreError) { - return false; - } - throw ex; - } - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java index 38e048ab1e..2d9f2d39ab 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java @@ -61,8 +61,8 @@ PreparedLayer build() { private PreparedLayer(Layer layer, String name, Optional stateInTarget) { this.layer = layer; - this.stateInTarget = stateInTarget; this.name = name; + this.stateInTarget = stateInTarget; } String getName() { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 2eae3e6fc7..5ce0a38727 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -40,9 +40,9 @@ /** Pulls and caches a single base image layer. */ class PullAndCacheBaseImageLayerStep implements Callable { - private interface LayerChecker { + private interface BlobChecker { - Optional exists(Layer layer) throws IOException, RegistryException; + Optional exists(DescriptorDigest digest) throws IOException, RegistryException; } private static final String DESCRIPTION = "Pulling base image layer %s"; @@ -109,18 +109,16 @@ private static boolean canSkipCachingBaseImageLayers( // .collect(Collectors.toList()); // } - static ImmutableList makeListForcedDownload( + static ImmutableList makeListForForcedDownload( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth) { + BlobChecker noOpBlobChecker = ignored -> Optional.empty(); return makeList( - buildConfiguration, - progressEventDispatcherFactory, - baseImageAndAuth, - ignored -> Optional.empty()); + buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, noOpBlobChecker); } - static ImmutableList makeListOptimized( + static ImmutableList makeListForSelectiveDownload( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth, @@ -132,21 +130,18 @@ static ImmutableList makeListOptimized( .newTargetImageRegistryClientFactory() .setAuthorization(pushAuthorization) .newRegistryClient(); - - LayerChecker layerChecker = - layer -> - Optional.of( - targetRegistryClient.checkBlob(layer.getBlobDescriptor().getDigest()).isPresent()); + BlobChecker blobChecker = + digest -> Optional.of(targetRegistryClient.checkBlob(digest).isPresent()); return makeList( - buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, layerChecker); + buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, blobChecker); } private static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth, - LayerChecker layerChecker) { + BlobChecker blobChecker) { ImmutableList baseImageLayers = baseImageAndAuth.getImage().getLayers(); try (ProgressEventDispatcher progressEventDispatcher = @@ -164,7 +159,7 @@ private static ImmutableList makeList( progressEventDispatcher.newChildProducer(), layer, baseImageAndAuth.getAuthorization(), - layerChecker)); + blobChecker)); } return ImmutableList.copyOf(layerPullers); } @@ -175,19 +170,19 @@ private static ImmutableList makeList( private final Layer layer; private final @Nullable Authorization pullAuthorization; - private final LayerChecker layerChecker; + private final BlobChecker blobChecker; PullAndCacheBaseImageLayerStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Layer layer, @Nullable Authorization pullAuthorization, - LayerChecker layerChecker) { + BlobChecker blobChecker) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.layer = layer; this.pullAuthorization = pullAuthorization; - this.layerChecker = layerChecker; + this.blobChecker = blobChecker; } @Override @@ -199,15 +194,15 @@ public PreparedLayer call() throws IOException, CacheCorruptedException, Registr new TimerEventDispatcher( buildConfiguration.getEventHandlers(), String.format(DESCRIPTION, layerDigest))) { - Optional layerExists = layerChecker.exists(layer); + Optional layerExists = blobChecker.exists(layerDigest); if (layerExists.orElse(false)) { return new PreparedLayer.Builder(layer).setStateInTarget(layerExists).build(); } Cache cache = buildConfiguration.getBaseImageLayersCache(); - Optional optionalCachedLayer = cache.retrieve(layerDigest); // Checks if the layer already exists in the cache. + Optional optionalCachedLayer = cache.retrieve(layerDigest); if (optionalCachedLayer.isPresent()) { CachedLayer cachedLayer = optionalCachedLayer.get(); return new PreparedLayer.Builder(cachedLayer).setStateInTarget(layerExists).build(); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java index ad56f22939..5b9c330e60 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.builder.steps; +import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.api.LogEvent; import com.google.cloud.tools.jib.api.RegistryException; import com.google.cloud.tools.jib.blob.Blob; @@ -62,9 +63,10 @@ class PushBlobStep implements Callable { @Override public BlobDescriptor call() throws IOException, RegistryException { EventHandlers eventHandlers = buildConfiguration.getEventHandlers(); + DescriptorDigest blobDigest = blobDescriptor.getDigest(); try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create( - "pushing blob " + blobDescriptor.getDigest(), blobDescriptor.getSize()); + "pushing blob " + blobDigest, blobDescriptor.getSize()); TimerEventDispatcher ignored = new TimerEventDispatcher(eventHandlers, DESCRIPTION + blobDescriptor); ThrottledAccumulatingConsumer throttledProgressReporter = @@ -76,7 +78,7 @@ public BlobDescriptor call() throws IOException, RegistryException { .newRegistryClient(); // check if the BLOB is available - if (doBlobCheck && registryClient.checkBlob(blobDescriptor.getDigest()).isPresent()) { + if (doBlobCheck && registryClient.checkBlob(blobDigest).isPresent()) { eventHandlers.dispatch( LogEvent.info("BLOB : " + blobDescriptor + " already exists on registry")); return blobDescriptor; @@ -90,8 +92,7 @@ public BlobDescriptor call() throws IOException, RegistryException { String baseRepository = buildConfiguration.getBaseImageConfiguration().getImageRepository(); String targetRegistry = buildConfiguration.getTargetImageConfiguration().getImageRegistry(); String sourceRepository = targetRegistry.equals(baseRegistry) ? baseRepository : null; - registryClient.pushBlob( - blobDescriptor.getDigest(), blob, sourceRepository, throttledProgressReporter); + registryClient.pushBlob(blobDigest, blob, sourceRepository, throttledProgressReporter); return blobDescriptor; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 3bf4c70301..6f45ee5701 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -265,11 +265,11 @@ private void pullAndCacheBaseImageLayers(boolean forcePull) { () -> scheduleCallables( forcePull - ? PullAndCacheBaseImageLayerStep.makeListForcedDownload( + ? PullAndCacheBaseImageLayerStep.makeListForForcedDownload( buildConfiguration, childProgressDispatcherFactory, results.baseImageAndAuth.get()) - : PullAndCacheBaseImageLayerStep.makeListOptimized( + : PullAndCacheBaseImageLayerStep.makeListForSelectiveDownload( buildConfiguration, childProgressDispatcherFactory, results.baseImageAndAuth.get(), diff --git a/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java b/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java index 645c6cb9af..a8c770b1ef 100644 --- a/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java +++ b/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java @@ -109,19 +109,22 @@ private static Verifier build( verifier.addCliOption("--file=" + pomXml); verifier.executeGoals(Arrays.asList("clean", "compile")); + if (!buildTwice) { + verifier.executeGoal("jib:build"); + return verifier; + } + // Builds twice, and checks if the second build took less time. + verifier.addCliOption("-Djib.forceDownload=true"); verifier.executeGoal("jib:build"); float timeOne = getBuildTimeFromVerifierLog(verifier); - if (buildTwice) { - verifier.resetStreams(); - verifier.executeGoal("jib:build"); - float timeTwo = getBuildTimeFromVerifierLog(verifier); - - String failMessage = "First build time (%s) is not greater than second build time (%s)"; - Assert.assertTrue(String.format(failMessage, timeOne, timeTwo), timeOne > timeTwo); - } + verifier.resetStreams(); + verifier.executeGoal("jib:build"); + float timeTwo = getBuildTimeFromVerifierLog(verifier); + String failMessage = "First build time (%s) is not greater than second build time (%s)"; + Assert.assertTrue(String.format(failMessage, timeOne, timeTwo), timeOne > timeTwo); return verifier; } From 86d7d3f48819ca22f7bebbcffbba0025a5b452af Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 11 Jul 2019 16:49:55 -0400 Subject: [PATCH 29/50] Clean up code --- .../BuildAndCacheApplicationLayerStep.java | 2 - .../steps/PullAndCacheBaseImageLayerStep.java | 66 +------------------ .../tools/jib/builder/steps/StepsRunner.java | 39 ----------- 3 files changed, 2 insertions(+), 105 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java index a802d5d81f..10dfc33153 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java @@ -95,8 +95,6 @@ public PreparedLayer call() throws IOException, CacheCorruptedException { try (ProgressEventDispatcher ignored = progressEventDispatcherFactory.create("building " + layerName + " layer", 1); TimerEventDispatcher ignored2 = new TimerEventDispatcher(eventHandlers, description)) { - // TODO: for registry push, check blob (layer) in the target registry to skip building layer. - Cache cache = buildConfiguration.getApplicationLayersCache(); // Don't build the layer if it exists already. diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 5ce0a38727..2737062263 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -40,75 +40,13 @@ /** Pulls and caches a single base image layer. */ class PullAndCacheBaseImageLayerStep implements Callable { - private interface BlobChecker { - - Optional exists(DescriptorDigest digest) throws IOException, RegistryException; - } - private static final String DESCRIPTION = "Pulling base image layer %s"; - private static boolean canSkipCachingBaseImageLayers( - BuildConfiguration buildConfiguration, - ImageAndAuthorization baseImageAndAuth, - @Nullable Authorization pushAuthorization) { - RegistryClient targetRegistryClient = - buildConfiguration - .newTargetImageRegistryClientFactory() - .setAuthorization(pushAuthorization) - .newRegistryClient(); + private interface BlobChecker { - try { - // TODO: parallelize - for (Layer layer : baseImageAndAuth.getImage().getLayers()) { - System.out.println(">>> Checking BLOb existence."); - System.out.println(">>> Size=" + layer.getBlobDescriptor().getSize()); - System.out.println(">>> Digest=" + layer.getBlobDescriptor().getDigest()); - System.out.println(">>> DiffId=" + layer.getDiffId()); - if (targetRegistryClient.checkBlob(layer.getBlobDescriptor().getDigest()) == null) { - System.out.println(">>> BLOb does not exist. Halt."); - return false; - } - } - System.out.println(">>> All BLObs exist. Can skip downloading base image layers."); - return true; - } catch (IOException | RegistryException ex) { - ex.printStackTrace(); - // fall through - } - System.out.println(">>> Exception occured."); - return false; + Optional exists(DescriptorDigest digest) throws IOException, RegistryException; } - // private static List createNoBlobCachedLayers( - // BuildConfiguration buildConfiguration, Image baseImage) - // throws IOException, CacheCorruptedException { - // // The image manifest is already saved, so we should delete it if not all of the layers are - // // actually cached. (--offline shouldn't see an incomplete caching state.) - // Cache cache = buildConfiguration.getBaseImageLayersCache(); - // for (Layer layer : baseImage.getLayers()) { - // if (!cache.retrieve(layer.getBlobDescriptor().getDigest()).isPresent()) { - // // TODO: delete the manifest. - // } - // } - // - // return baseImage - // .getLayers() - // .stream() - // .map( - // layer -> - // CachedLayer.builder() - // .setLayerDigest(layer.getBlobDescriptor().getDigest()) - // .setLayerSize(layer.getBlobDescriptor().getSize()) - // .setLayerDiffId(layer.getDiffId()) - // .setLayerBlob( - // ignored -> { - // throw new LayerPropertyNotFoundException("No actual BLOb attached"); - // }) - // .build()) - // .map(cachedLayer -> new PreparedLayer(cachedLayer, true, null)) - // .collect(Collectors.toList()); - // } - static ImmutableList makeListForForcedDownload( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 6f45ee5701..1270f24122 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -69,10 +69,6 @@ private static Future failedFuture() { private Future>> applicationLayerPushResults = failedFuture(); private Future containerConfigurationPushResult = failedFuture(); private Future buildResult = failedFuture(); - // - // private Future>> baseImageLayersExistInTargetRegistry = - // failedFuture(); - // private Future canSkipCachingBaseImageLayers = Futures.immediateFuture(false); } /** @@ -221,41 +217,6 @@ private void pullBaseImage() { new PullBaseImageStep(buildConfiguration, childProgressDispatcherFactory)); } - // private void checkBaseImageLayersInTargetRegistry() { - // ProgressEventDispatcher.Factory childProgressDispatcherFactory = - // Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); - // - // results.baseImageLayersExistInTargetRegistry = - // executorService.submit( - // () -> - // scheduleCallables( - // CheckBlobStep.makeList( - // buildConfiguration, - // childProgressDispatcherFactory, - // results.baseImageAndAuth.get().getImage(), - // results.pushAuthorization.get().orElse(null)))); - // } - - // private void checkCanSkipCachingBaseImageLayers() { - // Verify.verifyNotNull(rootProgressDispatcher).dispatchProgress(1); - // - // results.canSkipCachingBaseImageLayers = - // executorService.submit( - // () -> - // PullAndCacheBaseImageLayerStep.canSkipCachingBaseImageLayers( - // buildConfiguration, - // results.baseImageAndAuth.get(), - // results.pushAuthorization.get().orElse(null))); - // } - - // private boolean canSkipPushingBaseImageLayers() throws InterruptedException, - // ExecutionException { - // // TODO: we can optimize to short-curcuit as soon as any future evaluates to false. - // List layersExist = - // realizeFutures(results.baseImageLayersExistInTargetRegistry.get()); - // return layersExist.stream().allMatch(Boolean.TRUE::equals); - // } - private void pullAndCacheBaseImageLayers(boolean forcePull) { ProgressEventDispatcher.Factory childProgressDispatcherFactory = Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); From 68d0b82bc833e9d0808bd262f800069b6bb0118f Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 11 Jul 2019 16:51:26 -0400 Subject: [PATCH 30/50] Revert --- .../cloud/tools/jib/image/LayerPropertyNotFoundException.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerPropertyNotFoundException.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerPropertyNotFoundException.java index 75524e0038..3424bbd741 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerPropertyNotFoundException.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerPropertyNotFoundException.java @@ -19,7 +19,7 @@ /** Exception thrown when accessing non-existent properties of layers. */ public class LayerPropertyNotFoundException extends RuntimeException { - public LayerPropertyNotFoundException(String message) { + LayerPropertyNotFoundException(String message) { super(message); } } From 9b51f30a5c3788afdd8d597ea8e0407c70939c29 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 11 Jul 2019 16:59:20 -0400 Subject: [PATCH 31/50] Add new system property --- .../google/cloud/tools/jib/builder/steps/StepsRunner.java | 3 ++- .../google/cloud/tools/jib/global/JibSystemProperties.java | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 1270f24122..d8a776b53b 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -146,12 +146,13 @@ public StepsRunner tarBuildSteps(Path outputPath) { public StepsRunner registryPushSteps() { rootProgressDescription = "building image to registry"; + boolean forcePull = JibSystemProperties.forceLayerDownload(); stepsToRun.add(this::retrieveTargetRegistryCredentials); stepsToRun.add(this::authenticatePush); stepsToRun.add(this::pullBaseImage); - stepsToRun.add(() -> pullAndCacheBaseImageLayers(Boolean.getBoolean("jib.forceDownload"))); + stepsToRun.add(() -> pullAndCacheBaseImageLayers(forcePull)); stepsToRun.add(this::buildAndCacheApplicationLayers); // stepsToRun.add(this::checkBaseImageLayersInTargetRegistry); stepsToRun.add(this::buildImage); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java b/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java index cf78836645..f87c36f197 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java @@ -32,6 +32,8 @@ public class JibSystemProperties { private static final String SERIALIZE = "jibSerialize"; + private static final String FORCE_LAYER_DOWNLOAD = "jib.forceDownload"; + private static final String DISABLE_USER_AGENT = "_JIB_DISABLE_USER_AGENT"; /** @@ -91,6 +93,10 @@ public static boolean isUserAgentEnabled() { return Strings.isNullOrEmpty(System.getProperty(DISABLE_USER_AGENT)); } + public static boolean forceLayerDownload() { + return Boolean.getBoolean(FORCE_LAYER_DOWNLOAD); + } + /** * Checks the {@code jib.httpTimeout} system property for invalid (non-integer or negative) * values. From 934e019640089bfc52dfee1e8154b7eeb0a23658 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 11 Jul 2019 18:23:56 -0400 Subject: [PATCH 32/50] Update Javadoc --- .../com/google/cloud/tools/jib/builder/steps/PreparedLayer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java index 2d9f2d39ab..f5de8fc05b 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java @@ -44,7 +44,7 @@ Builder setName(String name) { return this; } - /** Sets whether the layer exists in a target destination. Empty means unknown. */ + /** Sets whether the layer exists in a target destination. Empty (absence) means unknown. */ Builder setStateInTarget(Optional stateInTarget) { this.stateInTarget = stateInTarget; return this; From 36c441a50ca1fa7c010434b931fd32c87e3f92eb Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 17 Jul 2019 14:39:43 -0400 Subject: [PATCH 33/50] Add PushBlobStepTest --- .../jib/builder/steps/PushBlobStepTest.java | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java new file mode 100644 index 0000000000..caaf53f088 --- /dev/null +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java @@ -0,0 +1,111 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.builder.steps; + +import com.google.cloud.tools.jib.api.DescriptorDigest; +import com.google.cloud.tools.jib.api.ImageReference; +import com.google.cloud.tools.jib.api.RegistryException; +import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; +import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.ImageConfiguration; +import com.google.cloud.tools.jib.event.EventHandlers; +import com.google.cloud.tools.jib.registry.RegistryClient; +import java.io.IOException; +import java.security.DigestException; +import java.util.Optional; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +/** Tests for {@link PushBlobStep}. */ +@RunWith(MockitoJUnitRunner.class) +public class PushBlobStepTest { + + private final ProgressEventDispatcher progressDispatcher = + ProgressEventDispatcher.newRoot(EventHandlers.NONE, "", 1); + private final ProgressEventDispatcher.Factory progressDispatcherFactory = + (ignored1, ignored2) -> progressDispatcher; + private BlobDescriptor blobDescriptor; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private BuildConfiguration buildConfiguration; + + @Mock private RegistryClient registryClient; + + @Before + public void setUp() throws DigestException { + String sha256Hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + blobDescriptor = new BlobDescriptor(154, DescriptorDigest.fromHash(sha256Hash)); + + RegistryClient.Factory registryClientFactory = + Mockito.mock(RegistryClient.Factory.class, Answers.RETURNS_SELF); + Mockito.when(registryClientFactory.newRegistryClient()).thenReturn(registryClient); + + Mockito.when(buildConfiguration.newTargetImageRegistryClientFactory()) + .thenReturn(registryClientFactory); + Mockito.when(buildConfiguration.getTargetImageConfiguration()) + .thenReturn(ImageConfiguration.builder(ImageReference.scratch()).build()); + } + + @After + public void tearDown() { + progressDispatcher.close(); + } + + @Test + public void testCall_doBlobCheckAndBlobExists() throws IOException, RegistryException { + Mockito.when(registryClient.checkBlob(Mockito.any())).thenReturn(Optional.of(blobDescriptor)); + + call(true); + + Mockito.verify(registryClient).checkBlob(Mockito.any()); + Mockito.verify(registryClient, Mockito.never()) + .pushBlob(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testCall_doBlobCheckAndBlobDoesNotExist() throws IOException, RegistryException { + Mockito.when(registryClient.checkBlob(Mockito.any())).thenReturn(Optional.empty()); + + call(true); + + Mockito.verify(registryClient).checkBlob(Mockito.any()); + Mockito.verify(registryClient) + .pushBlob(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testCall_noBlobCheck() throws IOException, RegistryException { + call(false); + + Mockito.verify(registryClient, Mockito.never()).checkBlob(Mockito.any()); + Mockito.verify(registryClient) + .pushBlob(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + private void call(boolean doBlobCheck) throws IOException, RegistryException { + new PushBlobStep( + buildConfiguration, progressDispatcherFactory, null, blobDescriptor, null, doBlobCheck) + .call(); + } +} From 1a2020e71c7f6ac1aa4fc7a21b6a89f3d566452e Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 17 Jul 2019 17:47:58 -0400 Subject: [PATCH 34/50] Add PullAndCacheBaseImageLayerStepTest --- .../PullAndCacheBaseImageLayerStepTest.java | 154 ++++++++++++++++++ .../jib/builder/steps/PushBlobStepTest.java | 28 +--- 2 files changed, 161 insertions(+), 21 deletions(-) create mode 100644 jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStepTest.java diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStepTest.java new file mode 100644 index 0000000000..4479e56335 --- /dev/null +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStepTest.java @@ -0,0 +1,154 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.builder.steps; + +import com.google.cloud.tools.jib.api.DescriptorDigest; +import com.google.cloud.tools.jib.api.RegistryException; +import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; +import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; +import com.google.cloud.tools.jib.cache.CacheCorruptedException; +import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.image.Image; +import com.google.cloud.tools.jib.image.Layer; +import com.google.cloud.tools.jib.image.ReferenceLayer; +import com.google.cloud.tools.jib.registry.RegistryClient; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.security.DigestException; +import java.util.Optional; +import java.util.function.Consumer; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.AdditionalAnswers; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer3; + +/** Tests for {@link PullAndCacheBaseImageLayerStep}. */ +@RunWith(MockitoJUnitRunner.class) +public class PullAndCacheBaseImageLayerStepTest { + + private ImageAndAuthorization baseImageAndAuth; + + private DescriptorDigest existingLayerDigest; + private DescriptorDigest freshLayerDigest; + + @Mock private Image image; + @Mock private RegistryClient registryClient; + + @Mock(answer = Answers.RETURNS_MOCKS) + private BuildConfiguration buildConfiguration; + + @Mock(answer = Answers.RETURNS_MOCKS) + private ProgressEventDispatcher.Factory progressDispatcherFactory; + + @Before + public void setUp() throws IOException, RegistryException, DigestException { + baseImageAndAuth = new ImageAndAuthorization(image, null); + + existingLayerDigest = + DescriptorDigest.fromHash( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + freshLayerDigest = + DescriptorDigest.fromHash( + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + + DescriptorDigest diffId = Mockito.mock(DescriptorDigest.class); + Layer existingLayer = new ReferenceLayer(new BlobDescriptor(existingLayerDigest), diffId); + Layer freshLayer = new ReferenceLayer(new BlobDescriptor(freshLayerDigest), diffId); + Mockito.when(image.getLayers()).thenReturn(ImmutableList.of(existingLayer, freshLayer)); + + Mockito.when(registryClient.checkBlob(existingLayerDigest)) + .thenReturn(Optional.of(Mockito.mock(BlobDescriptor.class))); + Mockito.when(registryClient.checkBlob(freshLayerDigest)).thenReturn(Optional.empty()); + + Answer3, Consumer> blobSizeListenerCaller = + (ignored1, blobSizeListener, ignored2) -> { + blobSizeListener.accept(Long.valueOf(12345)); + return null; + }; + Mockito.when(registryClient.pullBlob(Mockito.any(), Mockito.any(), Mockito.any())) + .thenAnswer(AdditionalAnswers.answer(blobSizeListenerCaller)); + + RegistryClient.Factory registryClientFactory = + Mockito.mock(RegistryClient.Factory.class, Answers.RETURNS_SELF); + Mockito.when(registryClientFactory.newRegistryClient()).thenReturn(registryClient); + + Mockito.lenient() + .when(buildConfiguration.newBaseImageRegistryClientFactory()) + .thenReturn(registryClientFactory); + Mockito.when(buildConfiguration.newTargetImageRegistryClientFactory()) + .thenReturn(registryClientFactory); + } + + @Test + public void testMakeListForSelectiveDownload() + throws IOException, CacheCorruptedException, RegistryException { + ImmutableList pullers = + PullAndCacheBaseImageLayerStep.makeListForSelectiveDownload( + buildConfiguration, progressDispatcherFactory, baseImageAndAuth, null); + + Assert.assertEquals(2, pullers.size()); + PreparedLayer preparedExistingLayer = pullers.get(0).call(); + PreparedLayer preparedFreshLayer = pullers.get(1).call(); + + Assert.assertTrue(preparedExistingLayer.existsInTarget().get()); + Assert.assertFalse(preparedFreshLayer.existsInTarget().get()); + + // Should have queried all blobs. + Mockito.verify(registryClient).checkBlob(existingLayerDigest); + Mockito.verify(registryClient).checkBlob(freshLayerDigest); + + // Only the missing layer should be pulled. + Mockito.verify(registryClient, Mockito.never()) + .pullBlob(Mockito.eq(existingLayerDigest), Mockito.any(), Mockito.any()); + Mockito.verify(registryClient) + .pullBlob(Mockito.eq(freshLayerDigest), Mockito.any(), Mockito.any()); + } + + @Test + public void testMakeListForForcedDownload() + throws IOException, CacheCorruptedException, RegistryException { + ImmutableList pullers = + PullAndCacheBaseImageLayerStep.makeListForForcedDownload( + buildConfiguration, progressDispatcherFactory, baseImageAndAuth); + + Assert.assertEquals(2, pullers.size()); + PreparedLayer preparedExistingLayer = pullers.get(0).call(); + PreparedLayer preparedFreshLayer = pullers.get(1).call(); + + // Unknown if layers exist in target registry. + Assert.assertEquals(Optional.empty(), preparedExistingLayer.existsInTarget()); + Assert.assertEquals(Optional.empty(), preparedFreshLayer.existsInTarget()); + + // No blob checking should happen. + Mockito.verify(registryClient, Mockito.never()).checkBlob(existingLayerDigest); + Mockito.verify(registryClient, Mockito.never()).checkBlob(freshLayerDigest); + + // All layers should be pulled. + Mockito.verify(registryClient) + .pullBlob(Mockito.eq(existingLayerDigest), Mockito.any(), Mockito.any()); + Mockito.verify(registryClient) + .pullBlob(Mockito.eq(freshLayerDigest), Mockito.any(), Mockito.any()); + } +} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java index caaf53f088..02ccffe64b 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java @@ -16,19 +16,15 @@ package com.google.cloud.tools.jib.builder.steps; -import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.api.ImageReference; import com.google.cloud.tools.jib.api.RegistryException; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.configuration.ImageConfiguration; -import com.google.cloud.tools.jib.event.EventHandlers; import com.google.cloud.tools.jib.registry.RegistryClient; import java.io.IOException; -import java.security.DigestException; import java.util.Optional; -import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -41,22 +37,17 @@ @RunWith(MockitoJUnitRunner.class) public class PushBlobStepTest { - private final ProgressEventDispatcher progressDispatcher = - ProgressEventDispatcher.newRoot(EventHandlers.NONE, "", 1); - private final ProgressEventDispatcher.Factory progressDispatcherFactory = - (ignored1, ignored2) -> progressDispatcher; - private BlobDescriptor blobDescriptor; + @Mock private BlobDescriptor blobDescriptor; + @Mock private RegistryClient registryClient; - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private BuildConfiguration buildConfiguration; + @Mock(answer = Answers.RETURNS_MOCKS) + private ProgressEventDispatcher.Factory progressDispatcherFactory; - @Mock private RegistryClient registryClient; + @Mock(answer = Answers.RETURNS_MOCKS) + private BuildConfiguration buildConfiguration; @Before - public void setUp() throws DigestException { - String sha256Hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - blobDescriptor = new BlobDescriptor(154, DescriptorDigest.fromHash(sha256Hash)); - + public void setUp() { RegistryClient.Factory registryClientFactory = Mockito.mock(RegistryClient.Factory.class, Answers.RETURNS_SELF); Mockito.when(registryClientFactory.newRegistryClient()).thenReturn(registryClient); @@ -67,11 +58,6 @@ public void setUp() throws DigestException { .thenReturn(ImageConfiguration.builder(ImageReference.scratch()).build()); } - @After - public void tearDown() { - progressDispatcher.close(); - } - @Test public void testCall_doBlobCheckAndBlobExists() throws IOException, RegistryException { Mockito.when(registryClient.checkBlob(Mockito.any())).thenReturn(Optional.of(blobDescriptor)); From e0b21e88e172f5c644bcf299326996d334844375 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Wed, 17 Jul 2019 18:24:26 -0400 Subject: [PATCH 35/50] Clean up --- .../jib/builder/steps/PushLayerStep.java | 4 +- .../PullAndCacheBaseImageLayerStepTest.java | 38 +++++++++++++++---- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java index 493a38ac9c..b0caa642af 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java @@ -83,8 +83,8 @@ public BlobDescriptor call() return layer.getBlobDescriptor(); } - // Boolean object not present --> unknown whether or not the layer exists in registry - boolean doBlobCheck = !layer.existsInTarget().isPresent(); + boolean doneBlobCheck = layer.existsInTarget().isPresent(); + boolean doBlobCheck = !doneBlobCheck; return new PushBlobStep( buildConfiguration, progressEventDispatcherFactory, diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStepTest.java index 4479e56335..84b77242a8 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStepTest.java @@ -82,14 +82,6 @@ public void setUp() throws IOException, RegistryException, DigestException { .thenReturn(Optional.of(Mockito.mock(BlobDescriptor.class))); Mockito.when(registryClient.checkBlob(freshLayerDigest)).thenReturn(Optional.empty()); - Answer3, Consumer> blobSizeListenerCaller = - (ignored1, blobSizeListener, ignored2) -> { - blobSizeListener.accept(Long.valueOf(12345)); - return null; - }; - Mockito.when(registryClient.pullBlob(Mockito.any(), Mockito.any(), Mockito.any())) - .thenAnswer(AdditionalAnswers.answer(blobSizeListenerCaller)); - RegistryClient.Factory registryClientFactory = Mockito.mock(RegistryClient.Factory.class, Answers.RETURNS_SELF); Mockito.when(registryClientFactory.newRegistryClient()).thenReturn(registryClient); @@ -99,6 +91,15 @@ public void setUp() throws IOException, RegistryException, DigestException { .thenReturn(registryClientFactory); Mockito.when(buildConfiguration.newTargetImageRegistryClientFactory()) .thenReturn(registryClientFactory); + + // necessary to prevent error from classes dealing with progress report + Answer3, Consumer> progressSizeSetter = + (ignored1, progressSizeConsumer, ignored2) -> { + progressSizeConsumer.accept(Long.valueOf(12345)); + return null; + }; + Mockito.when(registryClient.pullBlob(Mockito.any(), Mockito.any(), Mockito.any())) + .thenAnswer(AdditionalAnswers.answer(progressSizeSetter)); } @Test @@ -151,4 +152,25 @@ public void testMakeListForForcedDownload() Mockito.verify(registryClient) .pullBlob(Mockito.eq(freshLayerDigest), Mockito.any(), Mockito.any()); } + + @Test + public void testLayerMissingInCacheInOfflineMode() + throws CacheCorruptedException, RegistryException { + Mockito.when(buildConfiguration.isOffline()).thenReturn(true); + + ImmutableList pullers = + PullAndCacheBaseImageLayerStep.makeListForForcedDownload( + buildConfiguration, progressDispatcherFactory, baseImageAndAuth); + try { + pullers.get(1).call(); + Assert.fail(); + } catch (IOException ex) { + Assert.assertEquals( + "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " + + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb. Rerun " + + "Jib in online mode with \"-Djib.forceDownload=true\" to re-download the base " + + "image layers.", + ex.getMessage()); + } + } } From d5c5cb88c416814ff6717a63121e20820ba40fc9 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 22 Jul 2019 15:57:18 -0400 Subject: [PATCH 36/50] Add integration test --- .../gradle/SingleProjectIntegrationTest.java | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java b/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java index 4598457ae0..4b062062c5 100644 --- a/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java +++ b/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java @@ -21,6 +21,8 @@ import com.google.cloud.tools.jib.registry.LocalRegistry; import com.google.common.base.Splitter; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.DigestException; import java.time.Instant; import org.gradle.testkit.runner.BuildResult; @@ -145,14 +147,15 @@ private static void assertExtraDirectoryDeprecationWarning(String pomXml) + "'jib.extraDirectories.permissions'")); } - private static String buildAndRunComplex( + private static void buildAndRunComplex( String imageReference, String username, String password, LocalRegistry targetRegistry) throws IOException, InterruptedException { + Path baseCache = simpleTestProject.getProjectRoot().resolve("build/jib-base-cache"); BuildResult buildResult = simpleTestProject.build( "clean", "jib", - "-Djib.useOnlyProjectCache=true", + "-Djib.baseImageCache=" + baseCache, "-Djib.console=plain", "-D_TARGET_IMAGE=" + imageReference, "-D_TARGET_USERNAME=" + username, @@ -167,7 +170,10 @@ private static String buildAndRunComplex( assertDockerInspect(imageReference); String history = new Command("docker", "history", imageReference).run(); Assert.assertThat(history, CoreMatchers.containsString("jib-gradle-plugin")); - return new Command("docker", "run", "--rm", imageReference).run(); + Assert.assertEquals( + "Hello, world. An argument.\nrwxr-xr-x\nrwxrwxrwx\nfoo\ncat\n" + + "-Xms512m\n-Xdebug\nenvvalue1\nenvvalue2\n", + new Command("docker", "run", "--rm", imageReference).run()); } @Before @@ -316,9 +322,7 @@ public void testDockerDaemon_simple_multipleExtraDirectoriesWithAlternativeConfi public void testBuild_complex() throws IOException, InterruptedException { String targetImage = "localhost:6000/compleximage:gradle" + System.nanoTime(); Instant beforeBuild = Instant.now(); - Assert.assertEquals( - "Hello, world. An argument.\nrwxr-xr-x\nrwxrwxrwx\nfoo\ncat\n-Xms512m\n-Xdebug\nenvvalue1\nenvvalue2\n", - buildAndRunComplex(targetImage, "testuser2", "testpassword2", localRegistry2)); + buildAndRunComplex(targetImage, "testuser2", "testpassword2", localRegistry2); assertSimpleCreationTimeIsAfter(beforeBuild, targetImage); assertWorkingDirectory("", targetImage); } @@ -327,9 +331,7 @@ public void testBuild_complex() throws IOException, InterruptedException { public void testBuild_complex_sameFromAndToRegistry() throws IOException, InterruptedException { String targetImage = "localhost:5000/compleximage:gradle" + System.nanoTime(); Instant beforeBuild = Instant.now(); - Assert.assertEquals( - "Hello, world. An argument.\nrwxr-xr-x\nrwxrwxrwx\nfoo\ncat\n-Xms512m\n-Xdebug\nenvvalue1\nenvvalue2\n", - buildAndRunComplex(targetImage, "testuser", "testpassword", localRegistry1)); + buildAndRunComplex(targetImage, "testuser", "testpassword", localRegistry1); assertSimpleCreationTimeIsAfter(beforeBuild, targetImage); assertWorkingDirectory("", targetImage); } @@ -356,6 +358,22 @@ public void testDockerDaemon_jarContainerization() simpleTestProject, targetImage, "build-jar-containerization.gradle")); } + @Test + public void testBuild_skipDownloadingBaseImageLayers() throws IOException, InterruptedException { + Path baseCacheLayersDirectory = + simpleTestProject.getProjectRoot().resolve("build/jib-base-cache/layers"); + String targetImage = "localhost:6000/simpleimage:gradle" + System.nanoTime(); + + buildAndRunComplex(targetImage, "testuser2", "testpassword2", localRegistry2); + // Base image layer tarballs exist. + Assert.assertTrue(Files.exists(baseCacheLayersDirectory)); + Assert.assertTrue(baseCacheLayersDirectory.toFile().list().length >= 2); + + buildAndRunComplex(targetImage, "testuser2", "testpassword2", localRegistry2); + // no base layers downloaded after "gradle clean jib ..." + Assert.assertFalse(Files.exists(baseCacheLayersDirectory)); + } + @Test public void testBuild_dockerClient() throws IOException, InterruptedException, DigestException { Assume.assumeFalse(System.getProperty("os.name").startsWith("Windows")); From 14ef045da7e7d325ab34e02c6b0eed79eed751d0 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 22 Jul 2019 17:35:51 -0400 Subject: [PATCH 37/50] doBlobCheck --> forcePush and flip values --- .../cloud/tools/jib/builder/steps/PushBlobStep.java | 8 ++++---- .../steps/PushContainerConfigurationStep.java | 2 +- .../cloud/tools/jib/builder/steps/PushLayerStep.java | 10 +++++----- .../tools/jib/builder/steps/PushBlobStepTest.java | 12 ++++++------ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java index 5b9c330e60..12bba2e04e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java @@ -43,7 +43,7 @@ class PushBlobStep implements Callable { @Nullable private final Authorization authorization; private final BlobDescriptor blobDescriptor; private final Blob blob; - private final boolean doBlobCheck; + private final boolean forcePush; PushBlobStep( BuildConfiguration buildConfiguration, @@ -51,13 +51,13 @@ class PushBlobStep implements Callable { @Nullable Authorization authorization, BlobDescriptor blobDescriptor, Blob blob, - boolean doBlobCheck) { + boolean forcePush) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.authorization = authorization; this.blobDescriptor = blobDescriptor; this.blob = blob; - this.doBlobCheck = doBlobCheck; + this.forcePush = forcePush; } @Override @@ -78,7 +78,7 @@ public BlobDescriptor call() throws IOException, RegistryException { .newRegistryClient(); // check if the BLOB is available - if (doBlobCheck && registryClient.checkBlob(blobDigest).isPresent()) { + if (!forcePush && registryClient.checkBlob(blobDigest).isPresent()) { eventHandlers.dispatch( LogEvent.info("BLOB : " + blobDescriptor + " already exists on registry")); return blobDescriptor; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java index 966b39574d..54049feaa7 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java @@ -68,7 +68,7 @@ public BlobDescriptor call() throws IOException, RegistryException { pushAuthorization, Digests.computeDigest(containerConfiguration), Blobs.from(containerConfiguration), - true) + false) .call(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java index b0caa642af..de654b26f1 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java @@ -79,19 +79,19 @@ static ImmutableList makeList( public BlobDescriptor call() throws IOException, RegistryException, ExecutionException, InterruptedException { PreparedLayer layer = preparedLayer.get(); - if (layer.existsInTarget().orElse(false)) { // skip pushing if known to exist in registry - return layer.getBlobDescriptor(); + + boolean queriedExistence = layer.existsInTarget().isPresent(); + if (queriedExistence && layer.existsInTarget().get()) { + return layer.getBlobDescriptor(); // skip pushing if known to exist in registry } - boolean doneBlobCheck = layer.existsInTarget().isPresent(); - boolean doBlobCheck = !doneBlobCheck; return new PushBlobStep( buildConfiguration, progressEventDispatcherFactory, pushAuthorization, layer.getBlobDescriptor(), layer.getBlob(), - doBlobCheck) + queriedExistence) .call(); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java index 02ccffe64b..514cc4b674 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java @@ -62,7 +62,7 @@ public void setUp() { public void testCall_doBlobCheckAndBlobExists() throws IOException, RegistryException { Mockito.when(registryClient.checkBlob(Mockito.any())).thenReturn(Optional.of(blobDescriptor)); - call(true); + call(false); Mockito.verify(registryClient).checkBlob(Mockito.any()); Mockito.verify(registryClient, Mockito.never()) @@ -73,7 +73,7 @@ public void testCall_doBlobCheckAndBlobExists() throws IOException, RegistryExce public void testCall_doBlobCheckAndBlobDoesNotExist() throws IOException, RegistryException { Mockito.when(registryClient.checkBlob(Mockito.any())).thenReturn(Optional.empty()); - call(true); + call(false); Mockito.verify(registryClient).checkBlob(Mockito.any()); Mockito.verify(registryClient) @@ -81,17 +81,17 @@ public void testCall_doBlobCheckAndBlobDoesNotExist() throws IOException, Regist } @Test - public void testCall_noBlobCheck() throws IOException, RegistryException { - call(false); + public void testCall_forcePushWithNoBlobCheck() throws IOException, RegistryException { + call(true); Mockito.verify(registryClient, Mockito.never()).checkBlob(Mockito.any()); Mockito.verify(registryClient) .pushBlob(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); } - private void call(boolean doBlobCheck) throws IOException, RegistryException { + private void call(boolean forcePush) throws IOException, RegistryException { new PushBlobStep( - buildConfiguration, progressDispatcherFactory, null, blobDescriptor, null, doBlobCheck) + buildConfiguration, progressDispatcherFactory, null, blobDescriptor, null, forcePush) .call(); } } From d41c7614c2128c232b5f927b8e3ee8b7830a5312 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 22 Jul 2019 17:36:23 -0400 Subject: [PATCH 38/50] renmaing and comments only --- .../steps/PullAndCacheBaseImageLayerStep.java | 19 ++++++++++--------- .../tools/jib/builder/steps/StepsRunner.java | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java index 2737062263..ee4dccfc34 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java @@ -42,7 +42,8 @@ class PullAndCacheBaseImageLayerStep implements Callable { private static final String DESCRIPTION = "Pulling base image layer %s"; - private interface BlobChecker { + @FunctionalInterface + private interface BlobExistenceChecker { Optional exists(DescriptorDigest digest) throws IOException, RegistryException; } @@ -51,7 +52,7 @@ static ImmutableList makeListForForcedDownload( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth) { - BlobChecker noOpBlobChecker = ignored -> Optional.empty(); + BlobExistenceChecker noOpBlobChecker = ignored -> Optional.empty(); return makeList( buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, noOpBlobChecker); } @@ -68,18 +69,18 @@ static ImmutableList makeListForSelectiveDownloa .newTargetImageRegistryClientFactory() .setAuthorization(pushAuthorization) .newRegistryClient(); - BlobChecker blobChecker = + BlobExistenceChecker blobExistenceChecker = digest -> Optional.of(targetRegistryClient.checkBlob(digest).isPresent()); return makeList( - buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, blobChecker); + buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, blobExistenceChecker); } private static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth, - BlobChecker blobChecker) { + BlobExistenceChecker blobExistenceChecker) { ImmutableList baseImageLayers = baseImageAndAuth.getImage().getLayers(); try (ProgressEventDispatcher progressEventDispatcher = @@ -97,7 +98,7 @@ private static ImmutableList makeList( progressEventDispatcher.newChildProducer(), layer, baseImageAndAuth.getAuthorization(), - blobChecker)); + blobExistenceChecker)); } return ImmutableList.copyOf(layerPullers); } @@ -108,19 +109,19 @@ private static ImmutableList makeList( private final Layer layer; private final @Nullable Authorization pullAuthorization; - private final BlobChecker blobChecker; + private final BlobExistenceChecker blobChecker; PullAndCacheBaseImageLayerStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Layer layer, @Nullable Authorization pullAuthorization, - BlobChecker blobChecker) { + BlobExistenceChecker blobExistenceChecker) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.layer = layer; this.pullAuthorization = pullAuthorization; - this.blobChecker = blobChecker; + this.blobChecker = blobExistenceChecker; } @Override diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index dacfe2008b..8cc9a5a76e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -123,7 +123,7 @@ public StepsRunner dockerLoadSteps(DockerClient dockerClient) { // build and cache stepsToRun.add(this::pullBaseImage); - stepsToRun.add(() -> pullAndCacheBaseImageLayers(true)); + stepsToRun.add(() -> pullAndCacheBaseImageLayers(true)); // always pull layers for docker builds stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); // load to Docker @@ -136,7 +136,7 @@ public StepsRunner tarBuildSteps(Path outputPath) { // build and cache stepsToRun.add(this::pullBaseImage); - stepsToRun.add(() -> pullAndCacheBaseImageLayers(true)); + stepsToRun.add(() -> pullAndCacheBaseImageLayers(true)); // always pull layers for tar builds stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); // create a tar From 26614958b4ff0ae9bc04992cafe3ec5e78c34f18 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Mon, 22 Jul 2019 18:17:00 -0400 Subject: [PATCH 39/50] Rename to ObtainBaseImageLayerStep --- ...erStep.java => ObtainBaseImageLayerStep.java} | 14 +++++++------- .../tools/jib/builder/steps/PreparedLayer.java | 4 ++-- .../tools/jib/builder/steps/StepsRunner.java | 4 ++-- ...st.java => ObtainBaseImageLayerStepTest.java} | 16 ++++++++-------- 4 files changed, 19 insertions(+), 19 deletions(-) rename jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/{PullAndCacheBaseImageLayerStep.java => ObtainBaseImageLayerStep.java} (93%) rename jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/{PullAndCacheBaseImageLayerStepTest.java => ObtainBaseImageLayerStepTest.java} (93%) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java similarity index 93% rename from jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java index ee4dccfc34..a4064f97ef 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java @@ -38,7 +38,7 @@ import javax.annotation.Nullable; /** Pulls and caches a single base image layer. */ -class PullAndCacheBaseImageLayerStep implements Callable { +class ObtainBaseImageLayerStep implements Callable { private static final String DESCRIPTION = "Pulling base image layer %s"; @@ -48,7 +48,7 @@ private interface BlobExistenceChecker { Optional exists(DescriptorDigest digest) throws IOException, RegistryException; } - static ImmutableList makeListForForcedDownload( + static ImmutableList makeListForForcedDownload( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth) { @@ -57,7 +57,7 @@ static ImmutableList makeListForForcedDownload( buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, noOpBlobChecker); } - static ImmutableList makeListForSelectiveDownload( + static ImmutableList makeListForSelectiveDownload( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth, @@ -76,7 +76,7 @@ static ImmutableList makeListForSelectiveDownloa buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, blobExistenceChecker); } - private static ImmutableList makeList( + private static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth, @@ -90,10 +90,10 @@ private static ImmutableList makeList( new TimerEventDispatcher( buildConfiguration.getEventHandlers(), "Preparing base image layer pullers")) { - List layerPullers = new ArrayList<>(); + List layerPullers = new ArrayList<>(); for (Layer layer : baseImageLayers) { layerPullers.add( - new PullAndCacheBaseImageLayerStep( + new ObtainBaseImageLayerStep( buildConfiguration, progressEventDispatcher.newChildProducer(), layer, @@ -111,7 +111,7 @@ private static ImmutableList makeList( private final @Nullable Authorization pullAuthorization; private final BlobExistenceChecker blobChecker; - PullAndCacheBaseImageLayerStep( + ObtainBaseImageLayerStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Layer layer, diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java index f5de8fc05b..dd02e8a2cd 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java @@ -24,8 +24,8 @@ /** * Layer prepared from {@link BuildAndCacheApplicationLayerStep} and {@link - * PullAndCacheBaseImageLayerStep} to hold information about either a base image layer or an - * application layer. + * ObtainBaseImageLayerStep} to hold information about either a base image layer or an application + * layer. */ class PreparedLayer implements Layer { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 8cc9a5a76e..2c66a1ce73 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -227,11 +227,11 @@ private void pullAndCacheBaseImageLayers(boolean forcePull) { () -> scheduleCallables( forcePull - ? PullAndCacheBaseImageLayerStep.makeListForForcedDownload( + ? ObtainBaseImageLayerStep.makeListForForcedDownload( buildConfiguration, childProgressDispatcherFactory, results.baseImageAndAuth.get()) - : PullAndCacheBaseImageLayerStep.makeListForSelectiveDownload( + : ObtainBaseImageLayerStep.makeListForSelectiveDownload( buildConfiguration, childProgressDispatcherFactory, results.baseImageAndAuth.get(), diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java similarity index 93% rename from jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStepTest.java rename to jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java index 84b77242a8..15d6e07f23 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java @@ -44,9 +44,9 @@ import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer3; -/** Tests for {@link PullAndCacheBaseImageLayerStep}. */ +/** Tests for {@link ObtainBaseImageLayerStep}. */ @RunWith(MockitoJUnitRunner.class) -public class PullAndCacheBaseImageLayerStepTest { +public class ObtainBaseImageLayerStepTest { private ImageAndAuthorization baseImageAndAuth; @@ -105,8 +105,8 @@ public void setUp() throws IOException, RegistryException, DigestException { @Test public void testMakeListForSelectiveDownload() throws IOException, CacheCorruptedException, RegistryException { - ImmutableList pullers = - PullAndCacheBaseImageLayerStep.makeListForSelectiveDownload( + ImmutableList pullers = + ObtainBaseImageLayerStep.makeListForSelectiveDownload( buildConfiguration, progressDispatcherFactory, baseImageAndAuth, null); Assert.assertEquals(2, pullers.size()); @@ -130,8 +130,8 @@ public void testMakeListForSelectiveDownload() @Test public void testMakeListForForcedDownload() throws IOException, CacheCorruptedException, RegistryException { - ImmutableList pullers = - PullAndCacheBaseImageLayerStep.makeListForForcedDownload( + ImmutableList pullers = + ObtainBaseImageLayerStep.makeListForForcedDownload( buildConfiguration, progressDispatcherFactory, baseImageAndAuth); Assert.assertEquals(2, pullers.size()); @@ -158,8 +158,8 @@ public void testLayerMissingInCacheInOfflineMode() throws CacheCorruptedException, RegistryException { Mockito.when(buildConfiguration.isOffline()).thenReturn(true); - ImmutableList pullers = - PullAndCacheBaseImageLayerStep.makeListForForcedDownload( + ImmutableList pullers = + ObtainBaseImageLayerStep.makeListForForcedDownload( buildConfiguration, progressDispatcherFactory, baseImageAndAuth); try { pullers.get(1).call(); From 004b3bbcd358bed123015722695adac50874c2ad Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 23 Jul 2019 11:56:09 -0400 Subject: [PATCH 40/50] jib.forceDownload --> jib.cacheBaseImage (property renamed) --- .../tools/jib/builder/steps/StepsRunner.java | 2 +- .../tools/jib/global/JibSystemProperties.java | 22 +++++++++--- .../jib/global/JibSystemPropertiesTest.java | 34 ++++++++++++++++++- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 2c66a1ce73..7893eaa615 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -146,7 +146,7 @@ public StepsRunner tarBuildSteps(Path outputPath) { public StepsRunner registryPushSteps() { rootProgressDescription = "building image to registry"; - boolean forcePull = JibSystemProperties.forceLayerDownload(); + boolean forcePull = JibSystemProperties.alwaysCacheBaseImage(); stepsToRun.add(this::retrieveTargetRegistryCredentials); stepsToRun.add(this::authenticatePush); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java b/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java index 16ce095dff..cd2ccd8337 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java @@ -27,13 +27,13 @@ public class JibSystemProperties { @VisibleForTesting static final String CROSS_REPOSITORY_BLOB_MOUNTS = "jib.blobMounts"; + @VisibleForTesting static final String CACHE_BASE_IMAGE = "jib.cacheBaseImage"; + @VisibleForTesting public static final String SEND_CREDENTIALS_OVER_HTTP = "sendCredentialsOverHttp"; private static final String SERIALIZE = "jibSerialize"; - private static final String FORCE_LAYER_DOWNLOAD = "jib.forceDownload"; - private static final String DISABLE_USER_AGENT = "_JIB_DISABLE_USER_AGENT"; /** @@ -93,8 +93,22 @@ public static boolean isUserAgentEnabled() { return Strings.isNullOrEmpty(System.getProperty(DISABLE_USER_AGENT)); } - public static boolean forceLayerDownload() { - return Boolean.getBoolean(FORCE_LAYER_DOWNLOAD); + /** + * Gets whether to always cache base image layers. Determined from the {@code jib.cacheBaseImage} + * system property. + * + * @return true if the property is set to {@code always}; false otherwise + */ + public static boolean alwaysCacheBaseImage() { + String property = System.getProperty(CACHE_BASE_IMAGE, "as-needed"); + if ("always".equals(property)) { + return true; + } + if ("as-needed".equals(property)) { + return false; + } + throw new IllegalArgumentException( + CACHE_BASE_IMAGE + " should be either \"as-needed\" or \"always\" but was: " + property); } /** diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java index 4fca87bef0..5a4b941bcd 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java @@ -37,6 +37,8 @@ public void setUp() { @After public void tearDown() { System.clearProperty(JibSystemProperties.HTTP_TIMEOUT); + System.clearProperty(JibSystemProperties.CROSS_REPOSITORY_BLOB_MOUNTS); + System.clearProperty(JibSystemProperties.CACHE_BASE_IMAGE); System.clearProperty("http.proxyPort"); System.clearProperty("https.proxyPort"); if (httpProxyPortSaved != null) { @@ -45,7 +47,6 @@ public void tearDown() { if (httpsProxyPortSaved != null) { System.setProperty("https.proxyPort", httpsProxyPortSaved); } - System.clearProperty(JibSystemProperties.CROSS_REPOSITORY_BLOB_MOUNTS); } @Test @@ -182,4 +183,35 @@ public void testUseBlobMounts_other() { System.setProperty(JibSystemProperties.CROSS_REPOSITORY_BLOB_MOUNTS, "nonbool"); Assert.assertFalse(JibSystemProperties.useCrossRepositoryBlobMounts()); } + + @Test + public void testAlwaysCacheBaseImage_always() { + System.setProperty(JibSystemProperties.CACHE_BASE_IMAGE, "always"); + Assert.assertTrue(JibSystemProperties.alwaysCacheBaseImage()); + } + + @Test + public void testAlwaysCacheBaseImage_asNeeded() { + System.setProperty(JibSystemProperties.CACHE_BASE_IMAGE, "as-needed"); + Assert.assertFalse(JibSystemProperties.alwaysCacheBaseImage()); + } + + @Test + public void testAlwaysCacheBaseImage_propertyUnset() { + System.clearProperty(JibSystemProperties.CACHE_BASE_IMAGE); + Assert.assertFalse(JibSystemProperties.alwaysCacheBaseImage()); + } + + @Test + public void testAlwaysCacheBaseImage_invalidValue() { + System.setProperty(JibSystemProperties.CACHE_BASE_IMAGE, "invalid value"); + try { + JibSystemProperties.alwaysCacheBaseImage(); + Assert.fail(); + } catch (IllegalArgumentException ex) { + Assert.assertEquals( + "jib.cacheBaseImage should be either \"as-needed\" or \"always\" but was: invalid value", + ex.getMessage()); + } + } } From 4c2a70efc23ce9bcf9054bef645b3873ae390e3a Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 23 Jul 2019 12:30:44 -0400 Subject: [PATCH 41/50] Update error message --- jib-core/CHANGELOG.md | 4 ---- .../tools/jib/builder/steps/ObtainBaseImageLayerStep.java | 4 ++-- .../tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java | 2 +- .../cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/jib-core/CHANGELOG.md b/jib-core/CHANGELOG.md index aed3c47aaa..d9505cf473 100644 --- a/jib-core/CHANGELOG.md +++ b/jib-core/CHANGELOG.md @@ -3,10 +3,6 @@ All notable changes to this project will be documented in this file. ## [unreleased] -### Added - -- Say something - ### Changed - `JibContainerBuilder#addDependencies` is now split into three methods: `addDependencies`, `addSnapshotDependencies`, `addProjectDependencies` ([#1773](https://github.com/GoogleContainerTools/jib/pull/1773)) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java index a4064f97ef..4e597059c1 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java @@ -149,8 +149,8 @@ public PreparedLayer call() throws IOException, CacheCorruptedException, Registr throw new IOException( "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " + layerDigest - + ". Rerun Jib in online mode with \"-Djib.forceDownload=true\" to re-download the " - + "base image layers."); + + ". Rerun Jib in online mode with \"-Djib.cacheBaseImage=always\" to re-download " + + "the base image layers."); } RegistryClient registryClient = diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java index 15d6e07f23..349a2969ea 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java @@ -168,7 +168,7 @@ public void testLayerMissingInCacheInOfflineMode() Assert.assertEquals( "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb. Rerun " - + "Jib in online mode with \"-Djib.forceDownload=true\" to re-download the base " + + "Jib in online mode with \"-Djib.cacheBaseImage=always\" to re-download the base " + "image layers.", ex.getMessage()); } diff --git a/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java b/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java index a8c770b1ef..b691fa6979 100644 --- a/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java +++ b/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java @@ -115,7 +115,7 @@ private static Verifier build( } // Builds twice, and checks if the second build took less time. - verifier.addCliOption("-Djib.forceDownload=true"); + verifier.addCliOption("-Djib.cacheBaseImage=always"); verifier.executeGoal("jib:build"); float timeOne = getBuildTimeFromVerifierLog(verifier); From 707d9201717b2c6b637c8752e39394af39d5765d Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 23 Jul 2019 12:57:30 -0400 Subject: [PATCH 42/50] Rename variable --- .../tools/jib/builder/steps/ObtainBaseImageLayerStep.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java index 4e597059c1..50ce13a320 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java @@ -109,7 +109,7 @@ private static ImmutableList makeList( private final Layer layer; private final @Nullable Authorization pullAuthorization; - private final BlobExistenceChecker blobChecker; + private final BlobExistenceChecker blobExistenceChecker; ObtainBaseImageLayerStep( BuildConfiguration buildConfiguration, @@ -121,7 +121,7 @@ private static ImmutableList makeList( this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.layer = layer; this.pullAuthorization = pullAuthorization; - this.blobChecker = blobExistenceChecker; + this.blobExistenceChecker = blobExistenceChecker; } @Override @@ -133,7 +133,7 @@ public PreparedLayer call() throws IOException, CacheCorruptedException, Registr new TimerEventDispatcher( buildConfiguration.getEventHandlers(), String.format(DESCRIPTION, layerDigest))) { - Optional layerExists = blobChecker.exists(layerDigest); + Optional layerExists = blobExistenceChecker.exists(layerDigest); if (layerExists.orElse(false)) { return new PreparedLayer.Builder(layer).setStateInTarget(layerExists).build(); } From 05973e24b9424c0ef015496ac3bda5ae68378412 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 23 Jul 2019 13:03:42 -0400 Subject: [PATCH 43/50] Remove stray comment --- .../com/google/cloud/tools/jib/builder/steps/StepsRunner.java | 1 - 1 file changed, 1 deletion(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 7893eaa615..0cbe07f86a 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -154,7 +154,6 @@ public StepsRunner registryPushSteps() { stepsToRun.add(this::pullBaseImage); stepsToRun.add(() -> pullAndCacheBaseImageLayers(forcePull)); stepsToRun.add(this::buildAndCacheApplicationLayers); - // stepsToRun.add(this::checkBaseImageLayersInTargetRegistry); stepsToRun.add(this::buildImage); // push to registry From 99142614550446b077a30813d604c68161f2f82b Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 23 Jul 2019 16:21:22 -0400 Subject: [PATCH 44/50] Rename method and variables --- .../cloud/tools/jib/builder/steps/StepsRunner.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index 0cbe07f86a..1a73e959c2 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -123,7 +123,7 @@ public StepsRunner dockerLoadSteps(DockerClient dockerClient) { // build and cache stepsToRun.add(this::pullBaseImage); - stepsToRun.add(() -> pullAndCacheBaseImageLayers(true)); // always pull layers for docker builds + stepsToRun.add(() -> obtainBaseImageLayers(true)); // always pull layers for docker builds stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); // load to Docker @@ -136,7 +136,7 @@ public StepsRunner tarBuildSteps(Path outputPath) { // build and cache stepsToRun.add(this::pullBaseImage); - stepsToRun.add(() -> pullAndCacheBaseImageLayers(true)); // always pull layers for tar builds + stepsToRun.add(() -> obtainBaseImageLayers(true)); // always pull layers for tar builds stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); // create a tar @@ -146,13 +146,13 @@ public StepsRunner tarBuildSteps(Path outputPath) { public StepsRunner registryPushSteps() { rootProgressDescription = "building image to registry"; - boolean forcePull = JibSystemProperties.alwaysCacheBaseImage(); + boolean layersRequiredLocally = JibSystemProperties.alwaysCacheBaseImage(); stepsToRun.add(this::retrieveTargetRegistryCredentials); stepsToRun.add(this::authenticatePush); stepsToRun.add(this::pullBaseImage); - stepsToRun.add(() -> pullAndCacheBaseImageLayers(forcePull)); + stepsToRun.add(() -> obtainBaseImageLayers(layersRequiredLocally)); stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); @@ -217,7 +217,7 @@ private void pullBaseImage() { new PullBaseImageStep(buildConfiguration, childProgressDispatcherFactory)); } - private void pullAndCacheBaseImageLayers(boolean forcePull) { + private void obtainBaseImageLayers(boolean layersRequiredLocally) { ProgressEventDispatcher.Factory childProgressDispatcherFactory = Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); @@ -225,7 +225,7 @@ private void pullAndCacheBaseImageLayers(boolean forcePull) { executorService.submit( () -> scheduleCallables( - forcePull + layersRequiredLocally ? ObtainBaseImageLayerStep.makeListForForcedDownload( buildConfiguration, childProgressDispatcherFactory, From 4d35db603797445a7eac029317456a7b0601ea67 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Thu, 25 Jul 2019 18:58:40 -0400 Subject: [PATCH 45/50] Address review comments --- jib-core/CHANGELOG.md | 2 ++ .../cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java | 1 + 2 files changed, 3 insertions(+) diff --git a/jib-core/CHANGELOG.md b/jib-core/CHANGELOG.md index d9505cf473..0c3c01159e 100644 --- a/jib-core/CHANGELOG.md +++ b/jib-core/CHANGELOG.md @@ -3,6 +3,8 @@ All notable changes to this project will be documented in this file. ## [unreleased] +### Added + ### Changed - `JibContainerBuilder#addDependencies` is now split into three methods: `addDependencies`, `addSnapshotDependencies`, `addProjectDependencies` ([#1773](https://github.com/GoogleContainerTools/jib/pull/1773)) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java index 50ce13a320..de85f08580 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java @@ -69,6 +69,7 @@ static ImmutableList makeListForSelectiveDownload( .newTargetImageRegistryClientFactory() .setAuthorization(pushAuthorization) .newRegistryClient(); + // TODO: also check if cross-repo blob mount is possible. BlobExistenceChecker blobExistenceChecker = digest -> Optional.of(targetRegistryClient.checkBlob(digest).isPresent()); From 5fd7baa48b6a20c913c8bdc489b58e6176a48abb Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Jul 2019 10:53:25 -0400 Subject: [PATCH 46/50] Go back to Boolean jib.alwaysCacheBaseImage --- .../steps/ObtainBaseImageLayerStep.java | 4 +-- .../tools/jib/global/JibSystemProperties.java | 16 +++------- .../steps/ObtainBaseImageLayerStepTest.java | 4 +-- .../jib/global/JibSystemPropertiesTest.java | 31 +++++++------------ .../maven/BuildImageMojoIntegrationTest.java | 2 +- 5 files changed, 21 insertions(+), 36 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java index de85f08580..735be6a676 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java @@ -150,8 +150,8 @@ public PreparedLayer call() throws IOException, CacheCorruptedException, Registr throw new IOException( "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " + layerDigest - + ". Rerun Jib in online mode with \"-Djib.cacheBaseImage=always\" to re-download " - + "the base image layers."); + + ". Rerun Jib in online mode with \"-Djib.alwaysCacheBaseImage=true\" to " + + "re-download the base image layers."); } RegistryClient registryClient = diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java b/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java index cd2ccd8337..4a97dcb82b 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java @@ -27,7 +27,7 @@ public class JibSystemProperties { @VisibleForTesting static final String CROSS_REPOSITORY_BLOB_MOUNTS = "jib.blobMounts"; - @VisibleForTesting static final String CACHE_BASE_IMAGE = "jib.cacheBaseImage"; + @VisibleForTesting static final String ALWAYS_CACHE_BASE_IMAGE = "jib.alwaysCacheBaseImage"; @VisibleForTesting public static final String SEND_CREDENTIALS_OVER_HTTP = "sendCredentialsOverHttp"; @@ -94,21 +94,13 @@ public static boolean isUserAgentEnabled() { } /** - * Gets whether to always cache base image layers. Determined from the {@code jib.cacheBaseImage} - * system property. + * Gets whether to always cache base image layers. Determined from the {@code + * jib.alwaysCacheBaseImage} system property. * * @return true if the property is set to {@code always}; false otherwise */ public static boolean alwaysCacheBaseImage() { - String property = System.getProperty(CACHE_BASE_IMAGE, "as-needed"); - if ("always".equals(property)) { - return true; - } - if ("as-needed".equals(property)) { - return false; - } - throw new IllegalArgumentException( - CACHE_BASE_IMAGE + " should be either \"as-needed\" or \"always\" but was: " + property); + return Boolean.getBoolean(ALWAYS_CACHE_BASE_IMAGE); } /** diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java index 349a2969ea..ab7025d47d 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java @@ -168,8 +168,8 @@ public void testLayerMissingInCacheInOfflineMode() Assert.assertEquals( "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb. Rerun " - + "Jib in online mode with \"-Djib.cacheBaseImage=always\" to re-download the base " - + "image layers.", + + "Jib in online mode with \"-Djib.alwaysCacheBaseImage=true\" to re-download the " + + "base image layers.", ex.getMessage()); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java index 5a4b941bcd..85996ac10b 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java @@ -38,7 +38,7 @@ public void setUp() { public void tearDown() { System.clearProperty(JibSystemProperties.HTTP_TIMEOUT); System.clearProperty(JibSystemProperties.CROSS_REPOSITORY_BLOB_MOUNTS); - System.clearProperty(JibSystemProperties.CACHE_BASE_IMAGE); + System.clearProperty(JibSystemProperties.ALWAYS_CACHE_BASE_IMAGE); System.clearProperty("http.proxyPort"); System.clearProperty("https.proxyPort"); if (httpProxyPortSaved != null) { @@ -185,33 +185,26 @@ public void testUseBlobMounts_other() { } @Test - public void testAlwaysCacheBaseImage_always() { - System.setProperty(JibSystemProperties.CACHE_BASE_IMAGE, "always"); - Assert.assertTrue(JibSystemProperties.alwaysCacheBaseImage()); + public void testAlwaysCacheBaseImage_undefined() { + System.clearProperty(JibSystemProperties.ALWAYS_CACHE_BASE_IMAGE); + Assert.assertFalse(JibSystemProperties.alwaysCacheBaseImage()); } @Test - public void testAlwaysCacheBaseImage_asNeeded() { - System.setProperty(JibSystemProperties.CACHE_BASE_IMAGE, "as-needed"); - Assert.assertFalse(JibSystemProperties.alwaysCacheBaseImage()); + public void testAlwaysCacheBaseImage_true() { + System.setProperty(JibSystemProperties.ALWAYS_CACHE_BASE_IMAGE, "true"); + Assert.assertTrue(JibSystemProperties.alwaysCacheBaseImage()); } @Test - public void testAlwaysCacheBaseImage_propertyUnset() { - System.clearProperty(JibSystemProperties.CACHE_BASE_IMAGE); + public void testAlwaysCacheBaseImage_false() { + System.setProperty(JibSystemProperties.ALWAYS_CACHE_BASE_IMAGE, "false"); Assert.assertFalse(JibSystemProperties.alwaysCacheBaseImage()); } @Test - public void testAlwaysCacheBaseImage_invalidValue() { - System.setProperty(JibSystemProperties.CACHE_BASE_IMAGE, "invalid value"); - try { - JibSystemProperties.alwaysCacheBaseImage(); - Assert.fail(); - } catch (IllegalArgumentException ex) { - Assert.assertEquals( - "jib.cacheBaseImage should be either \"as-needed\" or \"always\" but was: invalid value", - ex.getMessage()); - } + public void testAlwaysCacheBaseImage_other() { + System.setProperty(JibSystemProperties.ALWAYS_CACHE_BASE_IMAGE, "nonbool"); + Assert.assertFalse(JibSystemProperties.alwaysCacheBaseImage()); } } diff --git a/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java b/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java index b691fa6979..a3e4f4ad50 100644 --- a/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java +++ b/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java @@ -115,7 +115,7 @@ private static Verifier build( } // Builds twice, and checks if the second build took less time. - verifier.addCliOption("-Djib.cacheBaseImage=always"); + verifier.addCliOption("-Djib.alwaysCacheBaseImage=true"); verifier.executeGoal("jib:build"); float timeOne = getBuildTimeFromVerifierLog(verifier); From 4f9e45991e9e97fb2f3a93e35f3a932fc7f02a49 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Jul 2019 11:21:38 -0400 Subject: [PATCH 47/50] Use openjdk8 on Travis instead of oraclejdk8 --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 21507dd4a1..4bbdfd7a38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,9 +4,8 @@ git: language: java matrix: include: - - name: Oracle JDK 8 - # using `openjdk8` leads to javadoc errors - jdk: oraclejdk8 + - name: OpenJDK 8 + jdk: openjdk8 - name: OpenJDK 11 jdk: openjdk11 env: From 21d734ec4de8f33848b45578d95092df8e23b0e4 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Jul 2019 12:00:34 -0400 Subject: [PATCH 48/50] Ubuntu Trusty --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4bbdfd7a38..83d1461b16 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,10 @@ git: language: java matrix: include: - - name: OpenJDK 8 - jdk: openjdk8 + # using `openjdk8` leads to javadoc errors. + - name: Oracle DK 8 on Trusty + jdk: oraclejdk8 + dist: trusty # oraclejdk8 is absent in Xenial, so go back to trusty - name: OpenJDK 11 jdk: openjdk11 env: From ded043aa521a24c5966ac2c2a9f8934a76a68023 Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Jul 2019 12:03:47 -0400 Subject: [PATCH 49/50] comments --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 83d1461b16..c5ac8dab8d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,10 +4,10 @@ git: language: java matrix: include: - # using `openjdk8` leads to javadoc errors. - - name: Oracle DK 8 on Trusty + - name: Oracle JDK 8 on Trusty + # using `openjdk8` leads to javadoc errors jdk: oraclejdk8 - dist: trusty # oraclejdk8 is absent in Xenial, so go back to trusty + dist: trusty # oraclejdk8 absent in Xenial - name: OpenJDK 11 jdk: openjdk11 env: From b0fe3cf6669580ac5656e16237c288c92ac503fc Mon Sep 17 00:00:00 2001 From: Chanseok Oh Date: Tue, 30 Jul 2019 13:38:11 -0400 Subject: [PATCH 50/50] Rename variable --- .../tools/jib/gradle/SingleProjectIntegrationTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java b/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java index 6b27ed3eb4..6ed601e0ab 100644 --- a/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java +++ b/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java @@ -364,18 +364,18 @@ public void testDockerDaemon_jarContainerization() @Test public void testBuild_skipDownloadingBaseImageLayers() throws IOException, InterruptedException { - Path baseCacheLayersDirectory = + Path baseLayersCacheDirectory = simpleTestProject.getProjectRoot().resolve("build/jib-base-cache/layers"); String targetImage = "localhost:6000/simpleimage:gradle" + System.nanoTime(); buildAndRunComplex(targetImage, "testuser2", "testpassword2", localRegistry2); // Base image layer tarballs exist. - Assert.assertTrue(Files.exists(baseCacheLayersDirectory)); - Assert.assertTrue(baseCacheLayersDirectory.toFile().list().length >= 2); + Assert.assertTrue(Files.exists(baseLayersCacheDirectory)); + Assert.assertTrue(baseLayersCacheDirectory.toFile().list().length >= 2); buildAndRunComplex(targetImage, "testuser2", "testpassword2", localRegistry2); // no base layers downloaded after "gradle clean jib ..." - Assert.assertFalse(Files.exists(baseCacheLayersDirectory)); + Assert.assertFalse(Files.exists(baseLayersCacheDirectory)); } @Test