Add ExtractTarStep and SaveDockerStep

jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java

@@ -23,6 +23,7 @@
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.util.zip.GZIPOutputStream;
 
 /** Static methods for {@link Blob}. */
 public class Blobs {
@@ -77,5 +78,20 @@ public static byte[] writeToByteArray(Blob blob) throws IOException {
  return byteArrayOutputStream.toByteArray();
  }
 
+ /**
+ * Gets a {@link Blob} that is {@code blob} compressed.
+ *
+ * @param blob the {@link Blob} to compress
+ * @return the compressed {@link Blob}
+ */
+ public static Blob compress(Blob blob) {
+ return Blobs.from(
+ outputStream -> {
+ try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream)) {
+ blob.writeTo(compressorStream);
+ }
+ });
+ }
+
  private Blobs() {}
 }

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java

@@ -0,0 +1,150 @@
+/*
+ * Copyright 2019 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.cloud.tools.jib.builder.steps;
+
+import com.fasterxml.jackson.databind.MapperFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.cloud.tools.jib.blob.Blob;
+import com.google.cloud.tools.jib.blob.BlobDescriptor;
+import com.google.cloud.tools.jib.blob.Blobs;
+import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage;
+import com.google.cloud.tools.jib.cache.CachedLayer;
+import com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate;
+import com.google.cloud.tools.jib.filesystem.FileOperations;
+import com.google.cloud.tools.jib.image.Image;
+import com.google.cloud.tools.jib.image.LayerCountMismatchException;
+import com.google.cloud.tools.jib.image.json.BadContainerConfigurationFormatException;
+import com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;
+import com.google.cloud.tools.jib.image.json.JsonToImageTranslator;
+import com.google.cloud.tools.jib.image.json.V22ManifestTemplate;
+import com.google.cloud.tools.jib.json.JsonTemplateMapper;
+import com.google.cloud.tools.jib.tar.TarExtractor;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.ByteStreams;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.zip.GZIPInputStream;
+
+/** Extracts a tar file base image. */
+public class ExtractTarStep implements Callable<LocalImage> {
+
+ /** Contains an {@link Image} and its layers. * */
+ static class LocalImage {
+ final Image baseImage;
+ final List<PreparedLayer> layers;
+
+ LocalImage(Image baseImage, List<PreparedLayer> layers) {
+ this.baseImage = baseImage;
+ this.layers = layers;
+ }
+ }
+
+ /**
+ * Checks the first two bytes of a file to see if it has been gzipped.
+ *
+ * @param path the file to check
+ * @return {@code true} if the file is gzipped, {@code false} if not
+ * @throws IOException if reading the file fails
+ * @see <a href="http://www.zlib.org/rfc-gzip.html#file-format">GZIP file format</a>
+ */
+ @VisibleForTesting
+ static boolean isGzipped(Path path) throws IOException {
+ try (InputStream inputStream = Files.newInputStream(path)) {
+ inputStream.mark(2);
+ int magic = (inputStream.read() & 0xff) | ((inputStream.read() << 8) & 0xff00);
+ return magic == GZIPInputStream.GZIP_MAGIC;
+ }
+ }
+
+ private final Path tarPath;
+ private final Path destination;
+
+ ExtractTarStep(Path tarPath, Path destination) {
+ this.tarPath = tarPath;
+ this.destination = destination;
+ }
+
+ @Override
+ public LocalImage call()
+ throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException {
+ Files.createDirectories(destination);
+ FileOperations.deleteRecursiveOnExit(destination);
+ TarExtractor.extract(tarPath, destination);
+
+ InputStream manifestStream = Files.newInputStream(destination.resolve("manifest.json"));
+ DockerManifestEntryTemplate loadManifest =
+ new ObjectMapper()
+ .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true)
+ .readValue(manifestStream, DockerManifestEntryTemplate[].class)[0];
+ manifestStream.close();
+ ContainerConfigurationTemplate configurationTemplate =
+ JsonTemplateMapper.readJsonFromFile(
+ destination.resolve(loadManifest.getConfig()), ContainerConfigurationTemplate.class);
+
+ List<String> layerFiles = loadManifest.getLayerFiles();
+ if (configurationTemplate.getLayerCount() != layerFiles.size()) {
+ throw new LayerCountMismatchException(
+ "Invalid base image format: manifest contains "
+ + layerFiles.size()
+ + " layers, but container configuration contains "
+ + configurationTemplate.getLayerCount()
+ + " layers");
+ }
+
+ // Check the first layer to see if the layers are compressed already. 'docker save' output is
+ // uncompressed, but a jib-built tar has compressed layers.
+ boolean layersAreCompressed =
+ layerFiles.size() > 0 && isGzipped(destination.resolve(layerFiles.get(0)));
+
+ // Process layer blobs
+ // TODO: Optimize; compressing/calculating layer digests is slow
+ List<PreparedLayer> layers = new ArrayList<>();
+ V22ManifestTemplate v22Manifest = new V22ManifestTemplate();
+ for (int index = 0; index < layerFiles.size(); index++) {
+ Path file = destination.resolve(layerFiles.get(index));
+
+ // Compress layers if necessary and calculate the digest/size
+ Blob blob = layersAreCompressed ? Blobs.from(file) : Blobs.compress(Blobs.from(file));
+ BlobDescriptor blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream());
+
+ // 'manifest' contains the layer files in the same order as the diff ids in 'configuration',
+ // so we don't need to recalculate those.
+ // https://containers.gitbook.io/build-containers-the-hard-way/#docker-load-format
+ CachedLayer layer =
+ CachedLayer.builder()
+ .setLayerBlob(blob)
+ .setLayerDigest(blobDescriptor.getDigest())
+ .setLayerSize(blobDescriptor.getSize())
+ .setLayerDiffId(configurationTemplate.getLayerDiffId(index))
+ .build();
+
+ layers.add(new PreparedLayer.Builder(layer).build());
+ v22Manifest.addLayer(blobDescriptor.getSize(), blobDescriptor.getDigest());
+ }
+
+ BlobDescriptor configDescriptor =
+ Blobs.from(configurationTemplate).writeTo(ByteStreams.nullOutputStream());
+ v22Manifest.setContainerConfiguration(configDescriptor.getSize(), configDescriptor.getDigest());
+ Image image = JsonToImageTranslator.toImage(v22Manifest, configurationTemplate);
+ return new LocalImage(image, layers);
+ }
+}

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.cloud.tools.jib.builder.steps;
+
+import com.google.cloud.tools.jib.api.ImageReference;
+import com.google.cloud.tools.jib.configuration.BuildConfiguration;
+import com.google.cloud.tools.jib.docker.DockerClient;
+import com.google.cloud.tools.jib.filesystem.FileOperations;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+
+/** Saves an image from the docker daemon. */
+public class SaveDockerStep implements Callable<Path> {
+
+ private final BuildConfiguration buildConfiguration;
+ private final DockerClient dockerClient;
+
+ SaveDockerStep(BuildConfiguration buildConfiguration, DockerClient dockerClient) {
+ this.buildConfiguration = buildConfiguration;
+ this.dockerClient = dockerClient;
+ }
+
+ @Override
+ public Path call() throws IOException, InterruptedException {
+ Path outputDir = Files.createTempDirectory("jib-docker-save");
+ FileOperations.deleteRecursiveOnExit(outputDir);
+ Path outputPath = outputDir.resolve("out.tar");
+ ImageReference imageReference = buildConfiguration.getBaseImageConfiguration().getImage();
+ dockerClient.save(imageReference, outputPath);
+ return outputPath;
+ }
+}

jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java

@@ -27,7 +27,7 @@
 public class CachedLayer implements Layer {
 
  /** Builds a {@link CachedLayer}. */
- static class Builder {
+ public static class Builder {
 
  @Nullable private DescriptorDigest layerDigest;
  @Nullable private DescriptorDigest layerDiffId;
@@ -36,22 +36,22 @@ static class Builder {
 
  private Builder() {}
 
- Builder setLayerDigest(DescriptorDigest layerDigest) {
+ public Builder setLayerDigest(DescriptorDigest layerDigest) {
  this.layerDigest = layerDigest;
  return this;
  }
 
- Builder setLayerDiffId(DescriptorDigest layerDiffId) {
+ public Builder setLayerDiffId(DescriptorDigest layerDiffId) {
  this.layerDiffId = layerDiffId;
  return this;
  }
 
- Builder setLayerSize(long layerSize) {
+ public Builder setLayerSize(long layerSize) {
  this.layerSize = layerSize;
  return this;
  }
 
- Builder setLayerBlob(Blob layerBlob) {
+ public Builder setLayerBlob(Blob layerBlob) {
  this.layerBlob = layerBlob;
  return this;
  }
@@ -60,7 +60,7 @@ boolean hasLayerBlob() {
  return layerBlob != null;
  }
 
- CachedLayer build() {
+ public CachedLayer build() {
  return new CachedLayer(
  Preconditions.checkNotNull(layerDigest, "layerDigest required"),
  Preconditions.checkNotNull(layerDiffId, "layerDiffId required"),
@@ -74,7 +74,7 @@ CachedLayer build() {
  *
  * @return the new {@link Builder}
  */
- static Builder builder() {
+ public static Builder builder() {
  return new Builder();
  }
 

jib-core/src/main/java/com/google/cloud/tools/jib/filesystem/FileOperations.java

@@ -17,6 +17,8 @@
 package com.google.cloud.tools.jib.filesystem;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.io.MoreFiles;
+import com.google.common.io.RecursiveDeleteOption;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.channels.Channels;
@@ -85,5 +87,24 @@ public static OutputStream newLockingOutputStream(Path file) throws IOException
  return Channels.newOutputStream(channel);
  }
 
+ /**
+ * Sets up a shutdown hook that tries to delete a file or directory.
+ *
+ * @param path the path to the file or directory
+ */
+ public static void deleteRecursiveOnExit(Path path) {
+ Runtime.getRuntime()
+ .addShutdownHook(
+ new Thread(
+ () -> {
+ if (Files.exists(path)) {
+ try {
+ MoreFiles.deleteRecursively(path, RecursiveDeleteOption.ALLOW_INSECURE);
+ } catch (IOException ignored) {
+ }
+ }
+ }));
+ }
+
  private FileOperations() {}
 }

jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplate.java

@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.google.cloud.tools.jib.api.DescriptorDigest;
 import com.google.cloud.tools.jib.json.JsonTemplate;
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import java.util.ArrayList;
 import java.util.List;
@@ -369,8 +368,11 @@ String getContainerUser() {
  return config.Volumes;
  }
 
- @VisibleForTesting
- DescriptorDigest getLayerDiffId(int index) {
+ public DescriptorDigest getLayerDiffId(int index) {
  return rootfs.diff_ids.get(index);
  }
+
+ public int getLayerCount() {
+ return rootfs.diff_ids.size();
+ }
 }