Adds support for creating Windows scratch-equivalent baselayers

.gitignore

@@ -13,3 +13,6 @@
 
 # IDEs
 .idea/
+
+# Build timestamp
+tools/bcdhive_generator/.build

Makefile

@@ -1,5 +1,6 @@
 # Go parameters
 GOCMD?=go
+GO_VERSION=$(shell go list -m -f "{{.GoVersion}}")
 PACKAGE_BASE=github.com/buildpacks/imgutil
 
 all: test
@@ -20,5 +21,22 @@ lint: install-golangci-lint
  @echo "> Linting code..."
  @golangci-lint run -c golangci.yaml
 
-test: format lint
+tools/bcdhive_generator/.build: $(wildcard tools/bcdhive_generator/*)
+ifneq ($(OS),Windows_NT)
+ @echo "> Building bcdhive-generator in Docker using current golang version"
+ docker build tools/bcdhive_generator --tag bcdhive-generator --build-arg go_version=$(GO_VERSION)
+
+ @touch tools/bcdhive_generator/.build
+else
+ @echo "> Cannot generate on Windows"
+endif
+
+layer/bcdhive_generated.go: layer/windows_baselayer.go tools/bcdhive_generator/.build
+ifneq ($(OS),Windows_NT)
+ $(GOCMD) generate ./...
+else
+ @echo "> Cannot generate on Windows"
+endif
+
+test: layer/bcdhive_generated.go format lint
  $(GOCMD) test -parallel=1 -count=1 -v ./...

go.sum

@@ -122,14 +122,6 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-containerregistry v0.0.0-20200311163244-4b1985e5ea21 h1:rz5VzU1xKHR8HDtifeAJ+SPwE0v3YW0AEien/Lobgww=

layer/windows_baselayer.go

@@ -0,0 +1,84 @@
+package layer
+
+import (
+ "archive/tar"
+ "bytes"
+ "io"
+)
+
+// Generate using `make generate`
+//go:generate docker run --rm -v $PWD:/out/ bcdhive-generator -file=/out/layer/bcdhive_generated.go -package=layer -func=BaseLayerBCD
+
+// Windows base layers must follow this pattern:
+// \-> UtilityVM/Files/EFI/Microsoft/Boot/BCD (file must exist and a valid BCD format - from bcdhive_gen)
+// \-> Files/Windows/System32/config/DEFAULT (file and must exist but can be empty)
+// \-> Files/Windows/System32/config/SAM (file must exist but can be empty)
+// \-> Files/Windows/System32/config/SECURITY (file must exist but can be empty)
+// \-> Files/Windows/System32/config/SOFTWARE (file must exist but can be empty)
+// \-> Files/Windows/System32/config/SYSTEM (file must exist but can be empty)
+// Refs:
+// https:/microsoft/hcsshim/blob/master/internal/wclayer/legacy.go
+// https:/moby/moby/blob/master/daemon/graphdriver/windows/windows.go#L48
+func WindowsBaseLayer() (io.Reader, error) {
+ bcdBytes, err := BaseLayerBCD()
+ if err != nil {
+ return nil, err
+ }
+
+ layerBuffer := &bytes.Buffer{}
+ tw := tar.NewWriter(layerBuffer)
+
+ if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM", Typeflag: tar.TypeDir}); err != nil {
+ return nil, err
+ }
+ if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files", Typeflag: tar.TypeDir}); err != nil {
+ return nil, err
+ }
+ if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI", Typeflag: tar.TypeDir}); err != nil {
+ return nil, err
+ }
+ if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft", Typeflag: tar.TypeDir}); err != nil {
+ return nil, err
+ }
+ if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft/Boot", Typeflag: tar.TypeDir}); err != nil {
+ return nil, err
+ }
+
+ if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft/Boot/BCD", Size: int64(len(bcdBytes)), Mode: 0644}); err != nil {
+ return nil, err
+ }
+ if _, err := tw.Write(bcdBytes); err != nil {
+ return nil, err
+ }
+
+ if err := tw.WriteHeader(&tar.Header{Name: "Files", Typeflag: tar.TypeDir}); err != nil {
+ return nil, err
+ }
+ if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows", Typeflag: tar.TypeDir}); err != nil {
+ return nil, err
+ }
+ if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32", Typeflag: tar.TypeDir}); err != nil {
+ return nil, err
+ }
+ if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config", Typeflag: tar.TypeDir}); err != nil {
+ return nil, err
+ }
+
+ if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/DEFAULT", Size: 0, Mode: 0644}); err != nil {
+ return nil, err
+ }
+ if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SAM", Size: 0, Mode: 0644}); err != nil {
+ return nil, err
+ }
+ if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SECURITY", Size: 0, Mode: 0644}); err != nil {
+ return nil, err
+ }
+ if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SOFTWARE", Size: 0, Mode: 0644}); err != nil {
+ return nil, err
+ }
+ if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SYSTEM", Size: 0, Mode: 0644}); err != nil {
+ return nil, err
+ }
+
+ return layerBuffer, nil
+}

local/local_test.go

@@ -1080,7 +1080,6 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
  repoName,
  dockerClient,
  local.FromBaseImage(repoName),
- local.WithPreviousImage(repoName),
  )
  h.AssertNil(t, err)
 

testhelpers/testhelpers.go

@@ -2,25 +2,23 @@ package testhelpers
 
 import (
  "archive/tar"
- "bytes"
- "compress/gzip"
  "context"
  "crypto/sha256"
- "encoding/base64"
  "encoding/hex"
  "fmt"
  "io"
  "io/ioutil"
  "math/rand"
  "net/http"
  "os"
- "path"
  "regexp"
  "strings"
  "sync"
  "testing"
  "time"
 
+ "github.com/buildpacks/imgutil/layer"
+
  dockertypes "github.com/docker/docker/api/types"
  dockercli "github.com/docker/docker/client"
  "github.com/google/go-cmp/cmp"
@@ -222,137 +220,78 @@ func CreateSingleFileTarReader(path, txt string) io.ReadCloser {
  pr, pw := io.Pipe()
 
  go func() {
- var err error
- defer func() {
+ // Use the regular tar.Writer, as this isn't a layer tar.
+ tw := tar.NewWriter(pw)
+
+ if err := tw.WriteHeader(&tar.Header{Name: path, Size: int64(len(txt)), Mode: 0644}); err != nil {
  pw.CloseWithError(err)
- }()
+ }
 
- tw := tar.NewWriter(pw)
- defer tw.Close()
+ if _, err := tw.Write([]byte(txt)); err != nil {
+ pw.CloseWithError(err)
+ }
 
- err = writeTarSingleFileLinux(tw, path, txt) // Use the Linux writer, as this isn't a layer tar.
+ if err := tw.Close(); err != nil {
+ pw.CloseWithError(err)
+ }
+
+ if err := pw.Close(); err != nil {
+ pw.CloseWithError(err)
+ }
  }()
 
  return pr
 }
 
-func CreateSingleFileLayerTar(layerPath, txt, osType string) (string, error) {
- tarFile, err := ioutil.TempFile("", "create-single-file-layer-tar-path")
- if err != nil {
- return "", err
- }
- defer tarFile.Close()
-
- tw := tar.NewWriter(tarFile)
- defer tw.Close()
+type layerWriter interface {
+ WriteHeader(*tar.Header) error
+ Write([]byte) (int, error)
+ Close() error
+}
 
- // regular Linux layer
- writeFunc := writeTarSingleFileLinux
+func getLayerWriter(osType string, file *os.File) layerWriter {
  if osType == "windows" {
- // regular Windows layer
- writeFunc = writeTarSingleFileWindows
+ return layer.NewWindowsWriter(file)
  }
+ return tar.NewWriter(file)
+}
 
- err = writeFunc(tw, layerPath, txt)
+func CreateSingleFileLayerTar(layerPath, txt, osType string) (string, error) {
+ tarFile, err := ioutil.TempFile("", "create-single-file-layer-tar-path")
  if err != nil {
  return "", err
  }
+ defer tarFile.Close()
 
- return tarFile.Name(), nil
-}
+ tw := getLayerWriter(osType, tarFile)
 
-func writeTarSingleFileLinux(tw *tar.Writer, layerPath, txt string) error {
  if err := tw.WriteHeader(&tar.Header{Name: layerPath, Size: int64(len(txt)), Mode: 0644}); err != nil {
- return err
+ return "", err
  }
 
  if _, err := tw.Write([]byte(txt)); err != nil {
- return err
+ return "", err
+ }
+
+ if err := tw.Close(); err != nil {
+ return "", err
  }
 
- return nil
+ return tarFile.Name(), nil
 }
 
-// WindowsBaseLayer returns a minimal windows base layer.
-// This base layer cannot use for running but can be used for saving to a Windows daemon and container creation.
-// Windows image layers must follow this pattern¹:
-// - base layer² (always required; tar file with relative paths without "/" prefix; all parent directories require own tar entries)
-// \-> Files/Windows/System32/config/DEFAULT (file must exist but can be empty)
-// \-> Files/Windows/System32/config/SAM (file must exist but can be empty)
-// \-> Files/Windows/System32/config/SECURITY (file must exist but can be empty)
-// \-> Files/Windows/System32/config/SOFTWARE (file must exist but can be empty)
-// \-> Files/Windows/System32/config/SYSTEM (file must exist but can be empty)
-// \-> UtilityVM/Files/EFI/Microsoft/Boot/BCD (file must exist and a valid BCD format - via `bcdedit` tool as below)
-// - normal or top layer (optional; tar file with relative paths without "/" prefix; all parent directories require own tar entries)
-// \-> Files/ (required directory entry)
-// \-> Files/mystuff.exe (optional container filesystem files - C:\mystuff.exe)
-// \-> Hives/ (required directory entry)
-// \-> Hives/DefaultUser_Delta (optional Windows reg hive delta; BCD format - HKEY_USERS\.DEFAULT additional content)
-// \-> Hives/Sam_Delta (optional Windows reg hive delta; BCD format - HKEY_LOCAL_MACHINE\SAM additional content)
-// \-> Hives/Security_Delta (optional Windows reg hive delta; BCD format - HKEY_LOCAL_MACHINE\SECURITY additional content)
-// \-> Hives/Software_Delta (optional Windows reg hive delta; BCD format - HKEY_LOCAL_MACHINE\SOFTWARE additional content)
-// \-> Hives/System_Delta (optional Windows reg hive delta; BCD format - HKEY_LOCAL_MACHINE\SYSTEM additional content)
-// 1. This was all discovered experimentally and should be considered an undocumented API, subject to change when the Windows Daemon internals change
-// 2. There are many other files in an "real" base layer but this is the minimum set which a Daemon can store and use to create an container
 func WindowsBaseLayer(t *testing.T) string {
  tarFile, err := ioutil.TempFile("", "windows-base-layer.tar")
  AssertNil(t, err)
+ defer tarFile.Close()
 
- tw := tar.NewWriter(tarFile)
-
- //Valid BCD file required, containing Windows Boot Manager and Windows Boot Loader sections
- //Note: Gzip/Base64 encoded only to inline the binary BCD file here
- //CMD: `bcdedit /createstore c:\output-bcd & bcdedit /create {6a6c1f1b-59d4-11ea-9438-9402e6abd998} /d buildpacks.io /application osloader /store c:\output-bcd & bcdedit /create {bootmgr} /store c:\output-bcd & bcdedit /set {bootmgr} default {6a6c1f1b-59d4-11ea-9438-9402e6abd998} /store c:\output-bcd & bcdedit /enum all /store c:\output-bcd`
- //BASH: `gzip --stdout --best output-bcd | base64`
- bcdGzipBase64 := "H4sIABeDWF4CA+1YTWgTQRR+m2zSFItGkaJQcKXgyZX8bNKklxatUvyJoh4qKjS7O7GxzQ9Jai210JtFQXrUW4/e2ostCF4EoSCCF6HHQi9FWsxJepH43s5us02XYlFBcL7l7U7evHnzzZtvoJ0Ke5A7BABkoU/f3qxvfZEkbPuBg9oKNcK8fQ/68LkHBvTiuwTjUIOy9VZBR68J++NZ/sXU/J2vR99d/thxdzOz0PqbYp63+CqFWuH7wg+LGwj8MfRuvnovqiAgICAgICAgICAgIPB/YETPF8H+/96Bcw9A7flGo1EcPQvrRVginw99Q6cAfHbsEDYwpEFt+s7Y306PuTrQMmziVq1UYTdLpRr5HmNsdRSg38+N8o5Z9w7yzCDlt8ceB+rT7HlPQB8cAYn/CND9hOLjUZaf3xIEjlGelhiJX2wE7gOfy7lQeG2tU4Gt7vZlZ50KNNd5I7B7ncSVvlc91tmGdl1/yIxaFbYxph4EmLXzq/2nd/KHpGZ+RfbO71XHM2hTyWzSiOaiuppIm5oajbKsmtbiKXxFYiyZ1c10OjUNUMccZZxkGDkMsKpBfBS/s68KPHlbX3Kv10HDBvnXkKezr06/Yu0CB90dUe5KvlzLl7icNjB2LOeDbYn30VqpJmvofzTaZo2d8/H6k11hk5lsgQH1n4cLMACRlofDi/eItJc3ueoS15SbdwhN3od33eItQQqDorFIhPOVacyMH5SwbPO9PVlmgy79Un3I2n9Rvych+Ff0+6Grc9ldF6c/7PfWV9hDX1Sji2OswIq1qrOPiz5eqxU/7/Oab8XvvQ+f5b37cBityzUf1RqhOfqgvkW5qQ+bj6UPHcYhj1U2oQxZMGAUqnAOPSWMI33Pyc3z5j7P7vM2GDzgeUubLJtKxgw1YZimqrGeiJo1jKiai8f0uKaZWk86Md3UPdWezuiqzMd66XZV9q7XRuDgunXr1AfhXToFuy4rgaZOup82dvFwdLIW/D2djAQ4t3qA961avMIWL8leA30v5SuFiWyFXSuZ+VyemV68KIdXfYYlbz1lXLxicUtPcec8zwa5z9EXxYbb9uqLeExBEnWVRGVFIYemgwoJSKPeNGxF8WHYr6JHgzik7FYEYuinkTpGpvFJwfQO/5ch8beGgICAgICAwL+BnwAgqcMAIAAA"
- bcdGzip, _ := base64.StdEncoding.DecodeString(bcdGzipBase64)
- bcdReader, _ := gzip.NewReader(bytes.NewBuffer(bcdGzip))
- bcdBytes, _ := ioutil.ReadAll(bcdReader)
-
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files", Typeflag: tar.TypeDir}))
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows", Typeflag: tar.TypeDir}))
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32", Typeflag: tar.TypeDir}))
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config", Typeflag: tar.TypeDir}))
-
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM", Typeflag: tar.TypeDir}))
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files", Typeflag: tar.TypeDir}))
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI", Typeflag: tar.TypeDir}))
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft", Typeflag: tar.TypeDir}))
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft/Boot", Typeflag: tar.TypeDir}))
-
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/DEFAULT", Size: 0, Mode: 0644}))
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SAM", Size: 0, Mode: 0644}))
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SECURITY", Size: 0, Mode: 0644}))
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SOFTWARE", Size: 0, Mode: 0644}))
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SYSTEM", Size: 0, Mode: 0644}))
-
- AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft/Boot/BCD", Size: int64(len(bcdBytes)), Mode: 0644}))
- _, err = tw.Write(bcdBytes)
+ baseLayer, err := layer.WindowsBaseLayer()
  AssertNil(t, err)
 
- return tarFile.Name()
-}
-
-func writeTarSingleFileWindows(tw *tar.Writer, containerPath, txt string) error {
- // root Windows layer directories
- if err := tw.WriteHeader(&tar.Header{Name: "Files", Typeflag: tar.TypeDir}); err != nil {
- return err
- }
- if err := tw.WriteHeader(&tar.Header{Name: "Hives", Typeflag: tar.TypeDir}); err != nil {
- return err
- }
-
- // prepend file entries with "Files"
- layerPath := path.Join("Files", containerPath)
- if err := tw.WriteHeader(&tar.Header{Name: layerPath, Size: int64(len(txt)), Mode: 0644}); err != nil {
- return err
- }
-
- if _, err := tw.Write([]byte(txt)); err != nil {
- return err
- }
+ _, err = io.Copy(tarFile, baseLayer)
+ AssertNil(t, err)
 
- return nil
+ return tarFile.Name()
 }
 
 func FetchManifestLayers(t *testing.T, repoName string) []string {

tools/bcdhive_generator/Dockerfile

@@ -0,0 +1,15 @@
+ARG go_version
+FROM golang:${go_version}-buster
+
+RUN apt update \
+ && apt install -y libhivex-dev libhivex-bin libwin-hivex-perl
+
+COPY . /src
+
+WORKDIR /src
+
+RUN go generate ./ \
+ && go test -parallel=1 -count=1 -v . \
+ && go install .
+
+ENTRYPOINT ["/go/bin/bcdhive_gen"]