From fcd1ef2b775d03f0d40aea203aeca8fa0c3a6d6c Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Wed, 25 Sep 2019 12:08:52 -0400 Subject: [PATCH 1/2] Parallelize base image compression --- .../jib/builder/steps/ExtractTarStep.java | 32 ++++++++++++++----- .../steps/ObtainBaseImageLayerStep.java | 2 +- .../tools/jib/builder/steps/StepsRunner.java | 1 + .../jib/builder/steps/ExtractTarStepTest.java | 11 +++++-- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index 05652ba3bd..8e996e292c 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -51,6 +51,9 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -85,16 +88,19 @@ static boolean isGzipped(Path path) throws IOException { } } + private final ExecutorService executorService; private final BuildConfiguration buildConfiguration; private final Path tarPath; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; private final TempDirectoryProvider tempDirectoryProvider; ExtractTarStep( + ExecutorService executorService, BuildConfiguration buildConfiguration, Path tarPath, ProgressEventDispatcher.Factory progressEventDispatcherFactory, TempDirectoryProvider tempDirectoryProvider) { + this.executorService = executorService; this.buildConfiguration = buildConfiguration; this.tarPath = tarPath; this.progressEventDispatcherFactory = progressEventDispatcherFactory; @@ -104,7 +110,7 @@ static boolean isGzipped(Path path) throws IOException { @Override public LocalImage call() throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException, - CacheCorruptedException { + ExecutionException, InterruptedException { Path destination = tempDirectoryProvider.newDirectory(); try (TimerEventDispatcher ignored = new TimerEventDispatcher( @@ -139,21 +145,31 @@ public LocalImage call() // Process layer blobs // TODO: Optimize; compressing/calculating layer digests is slow - // e.g. parallelize, faster compression method + // e.g. faster compression method try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create( "processing base image layers", layerFiles.size())) { List layers = new ArrayList<>(layerFiles.size()); V22ManifestTemplate v22Manifest = new V22ManifestTemplate(); + // Start compressing layers in parallel + List> cachedLayers = new ArrayList<>(); for (int index = 0; index < layerFiles.size(); index++) { Path layerFile = destination.resolve(layerFiles.get(index)); - CachedLayer layer = - getCachedTarLayer( - configurationTemplate.getLayerDiffId(index), - layerFile, - layersAreCompressed, - progressEventDispatcher.newChildProducer()); + DescriptorDigest diffId = configurationTemplate.getLayerDiffId(index); + ProgressEventDispatcher.Factory progressEventDispatcherFactory = + progressEventDispatcher.newChildProducer(); + + cachedLayers.add( + executorService.submit( + () -> + getCachedTarLayer( + diffId, layerFile, layersAreCompressed, progressEventDispatcherFactory))); + } + + // Collect compressed layers and add to manifest + for (Future layerFuture : cachedLayers) { + CachedLayer layer = layerFuture.get(); layers.add(new PreparedLayer.Builder(layer).build()); v22Manifest.addLayer(layer.getSize(), layer.getDigest()); } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java index 6dff150ad4..251f9c298d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java @@ -116,7 +116,7 @@ private static ImmutableList makeList( private final @Nullable Authorization pullAuthorization; private final BlobExistenceChecker blobExistenceChecker; - ObtainBaseImageLayerStep( + private ObtainBaseImageLayerStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Layer layer, diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index bf447eca5b..a519e0ab0b 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -257,6 +257,7 @@ private void extractTar() { executorService.submit( () -> new ExtractTarStep( + executorService, buildConfiguration, results.tarPath.get(), childProgressDispatcherFactory, diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java index cdcc251f13..c5cdf42d1a 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java @@ -19,17 +19,18 @@ import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage; import com.google.cloud.tools.jib.cache.Cache; -import com.google.cloud.tools.jib.cache.CacheCorruptedException; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.EventHandlers; import com.google.cloud.tools.jib.filesystem.TempDirectoryProvider; import com.google.cloud.tools.jib.image.LayerCountMismatchException; import com.google.cloud.tools.jib.image.json.BadContainerConfigurationFormatException; import com.google.common.io.Resources; +import com.google.common.util.concurrent.MoreExecutors; import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.concurrent.ExecutionException; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -71,10 +72,12 @@ public void setup() throws IOException { @Test public void testCall_validDocker() throws URISyntaxException, LayerCountMismatchException, - BadContainerConfigurationFormatException, IOException, CacheCorruptedException { + BadContainerConfigurationFormatException, IOException, ExecutionException, + InterruptedException { Path dockerBuild = getResource("core/extraction/docker-save.tar"); LocalImage result = new ExtractTarStep( + MoreExecutors.newDirectExecutorService(), buildConfiguration, dockerBuild, progressEventDispatcherFactory, @@ -101,10 +104,12 @@ public void testCall_validDocker() @Test public void testCall_validTar() throws URISyntaxException, LayerCountMismatchException, - BadContainerConfigurationFormatException, IOException, CacheCorruptedException { + BadContainerConfigurationFormatException, IOException, ExecutionException, + InterruptedException { Path tarBuild = getResource("core/extraction/jib-image.tar"); LocalImage result = new ExtractTarStep( + MoreExecutors.newDirectExecutorService(), buildConfiguration, tarBuild, progressEventDispatcherFactory, From 457b81b793e4ac69c96b3b3022fe24017be3df10 Mon Sep 17 00:00:00 2001 From: "tad.cordle@gmail.com" Date: Wed, 25 Sep 2019 12:14:37 -0400 Subject: [PATCH 2/2] Changelog --- jib-core/CHANGELOG.md | 2 ++ jib-gradle-plugin/CHANGELOG.md | 2 ++ jib-maven-plugin/CHANGELOG.md | 2 ++ 3 files changed, 6 insertions(+) diff --git a/jib-core/CHANGELOG.md b/jib-core/CHANGELOG.md index 528b3ef581..0cb95d39b6 100644 --- a/jib-core/CHANGELOG.md +++ b/jib-core/CHANGELOG.md @@ -7,6 +7,8 @@ All notable changes to this project will be documented in this file. ### Changed +- Local base image layers are now processed in parallel, speeding up builds using large local base images. ([#1913](https://github.com/GoogleContainerTools/jib/issues/1913)) + ### Fixed - Fixed temporary directory cleanup during builds using local base images. ([#2016](https://github.com/GoogleContainerTools/jib/issues/2016)) diff --git a/jib-gradle-plugin/CHANGELOG.md b/jib-gradle-plugin/CHANGELOG.md index 82d0198fe7..48ae0e766e 100644 --- a/jib-gradle-plugin/CHANGELOG.md +++ b/jib-gradle-plugin/CHANGELOG.md @@ -7,6 +7,8 @@ All notable changes to this project will be documented in this file. ### Changed +- Local base image layers are now processed in parallel, speeding up builds using large local base images. ([#1913](https://github.com/GoogleContainerTools/jib/issues/1913)) + ### Fixed - Fixed temporary directory cleanup during builds using local base images. ([#2016](https://github.com/GoogleContainerTools/jib/issues/2016)) diff --git a/jib-maven-plugin/CHANGELOG.md b/jib-maven-plugin/CHANGELOG.md index ffd5185d58..a518ca19ba 100644 --- a/jib-maven-plugin/CHANGELOG.md +++ b/jib-maven-plugin/CHANGELOG.md @@ -7,6 +7,8 @@ All notable changes to this project will be documented in this file. ### Changed +- Local base image layers are now processed in parallel, speeding up builds using large local base images. ([#1913](https://github.com/GoogleContainerTools/jib/issues/1913)) + ### Fixed - Fixed temporary directory cleanup during builds using local base images. ([#2016](https://github.com/GoogleContainerTools/jib/issues/2016))