Merge pull request #2309 from tonistiigi/v0.13.0-picks

[v0.13] cherry-picks for v0.13.0

commands/bake.go

@@ -72,11 +72,9 @@ func runBake(ctx context.Context, dockerCli command.Cli, targets []string, in ba
 
  overrides := in.overrides
  if in.exportPush {
- if in.exportLoad {
- return errors.Errorf("push and load may not be set together at the moment")
- }
- overrides = append(overrides, "*.push=true")
- } else if in.exportLoad {
+ overrides = append(overrides, "*.output=type=registry")
+ }
+ if in.exportLoad {
  overrides = append(overrides, "*.output=type=docker")
  }
  if cFlags.noCache != nil {

controller/build/build.go

@@ -99,38 +99,38 @@ func RunBuild(ctx context.Context, dockerCli command.Cli, in controllerapi.Build
  return nil, nil, err
  }
  if in.ExportPush {
- if in.ExportLoad {
- return nil, nil, errors.Errorf("push and load may not be set together at the moment")
+ var pushUsed bool
+ for i := range outputs {
+ if outputs[i].Type == client.ExporterImage {
+ outputs[i].Attrs["push"] = "true"
+ pushUsed = true
+ }
  }
- if len(outputs) == 0 {
- outputs = []client.ExportEntry{{
- Type: "image",
+ if !pushUsed {
+ outputs = append(outputs, client.ExportEntry{
+ Type: client.ExporterImage,
  Attrs: map[string]string{
  "push": "true",
  },
- }}
- } else {
- switch outputs[0].Type {
- case "image":
- outputs[0].Attrs["push"] = "true"
- default:
- return nil, nil, errors.Errorf("push and %q output can't be used together", outputs[0].Type)
- }
+ })
  }
  }
  if in.ExportLoad {
- if len(outputs) == 0 {
- outputs = []client.ExportEntry{{
- Type: "docker",
- Attrs: map[string]string{},
- }}
- } else {
- switch outputs[0].Type {
- case "docker":
- default:
- return nil, nil, errors.Errorf("load and %q output can't be used together", outputs[0].Type)
+ var loadUsed bool
+ for i := range outputs {
+ if outputs[i].Type == client.ExporterDocker {
+ if _, ok := outputs[i].Attrs["dest"]; !ok {
+ loadUsed = true
+ break
+ }
  }
  }
+ if !loadUsed {
+ outputs = append(outputs, client.ExportEntry{
+ Type: client.ExporterDocker,
+ Attrs: map[string]string{},
+ })
+ }
  }
 
  annotations, err := buildflags.ParseAnnotations(in.Annotations)

go.mod

@@ -25,7 +25,7 @@ require (
  github.com/google/uuid v1.6.0
  github.com/hashicorp/go-cty-funcs v0.0.0-20230405223818-a090f58aa992
  github.com/hashicorp/hcl/v2 v2.19.1
- github.com/moby/buildkit v0.13.0-rc3
+ github.com/moby/buildkit v0.13.0
  github.com/moby/sys/mountinfo v0.7.1
  github.com/moby/sys/signal v0.7.0
  github.com/morikuni/aec v1.0.0
@@ -140,7 +140,7 @@ require (
  github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect
  github.com/shibumi/go-pathspec v1.3.0 // indirect
  github.com/theupdateframework/notary v0.7.0 // indirect
- github.com/tonistiigi/fsutil v0.0.0-20240223190444-7a889f53dbf6 // indirect
+ github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 // indirect
  github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect
  github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531 // indirect
  github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect

go.sum

@@ -311,8 +311,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
-github.com/moby/buildkit v0.13.0-rc3 h1:Dns5Ixnv4OH1MyLZy7S4G2m+MLwp2YEqrPPzEzfw7Tw=
-github.com/moby/buildkit v0.13.0-rc3/go.mod h1:5pRtk7Wuv929XRIp9tqPdq07mrnBpXAUoOYYfOj0nhA=
+github.com/moby/buildkit v0.13.0 h1:reVR1Y+rbNIUQ9jf0Q1YZVH5a/nhOixZsl+HJ9qQEGI=
+github.com/moby/buildkit v0.13.0/go.mod h1:aNmNQKLBFYAOFuzQjR3VA27/FijlvtBD1pjNwTSN37k=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
@@ -450,8 +450,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c=
 github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw=
-github.com/tonistiigi/fsutil v0.0.0-20240223190444-7a889f53dbf6 h1:v9u6pmdUkarXL/1S/6LGcG9wsiBLd9N/WyJq/Y9WPcg=
-github.com/tonistiigi/fsutil v0.0.0-20240223190444-7a889f53dbf6/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM=
+github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5 h1:oZS8KCqAg62sxJkEq/Ppzqrb6EooqzWtL8Oaex7bc5c=
+github.com/tonistiigi/fsutil v0.0.0-20240301111122-7525a1af2bb5/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM=
 github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0=
 github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk=
 github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531 h1:Y/M5lygoNPKwVNLMPXgVfsRT40CSFKXCxuU8LoHySjs=

tests/bake.go

@@ -2,13 +2,19 @@ package tests
 
 import (
  "encoding/json"
+ "fmt"
  "os"
  "path/filepath"
+ "strings"
  "testing"
 
  "github.com/containerd/continuity/fs/fstest"
  "github.com/docker/buildx/util/gitutil"
+ "github.com/moby/buildkit/identity"
+ "github.com/moby/buildkit/util/contentutil"
+ "github.com/moby/buildkit/util/testutil"
  "github.com/moby/buildkit/util/testutil/integration"
+ "github.com/pkg/errors"
  "github.com/stretchr/testify/require"
 )
 
@@ -36,6 +42,8 @@ var bakeTests = []func(t *testing.T, sb integration.Sandbox){
  testBakeShmSize,
  testBakeUlimits,
  testBakeRefs,
+ testBakeMultiExporters,
+ testBakeLoadPush,
 }
 
 func testBakeLocal(t *testing.T, sb integration.Sandbox) {
@@ -631,3 +639,155 @@ target "default" {
 
  require.NotEmpty(t, md.Default.BuildRef)
 }
+
+func testBakeMultiExporters(t *testing.T, sb integration.Sandbox) {
+ if sb.Name() != "docker" {
+ t.Skip("skipping test for non-docker workers")
+ }
+
+ registry, err := sb.NewRegistry()
+ if errors.Is(err, integration.ErrRequirements) {
+ t.Skip(err.Error())
+ }
+ require.NoError(t, err)
+
+ targetReg := registry + "/buildx/registry:latest"
+ targetStore := "buildx:local-" + identity.NewID()
+
+ var builderName string
+ t.Cleanup(func() {
+ if builderName == "" {
+ return
+ }
+
+ cmd := dockerCmd(sb, withArgs("image", "rm", targetStore))
+ cmd.Stderr = os.Stderr
+ require.NoError(t, cmd.Run())
+
+ out, err := rmCmd(sb, withArgs(builderName))
+ require.NoError(t, err, out)
+ })
+
+ // TODO: use stable buildkit image when v0.13.0 released
+ out, err := createCmd(sb, withArgs(
+ "--driver", "docker-container",
+ "--buildkitd-flags=--allow-insecure-entitlement=network.host",
+ "--driver-opt", "network=host",
+ "--driver-opt", "image=moby/buildkit:v0.13.0-rc3",
+ ))
+ require.NoError(t, err, out)
+ builderName = strings.TrimSpace(out)
+
+ dockerfile := []byte(`
+FROM scratch
+COPY foo /foo
+ `)
+ bakefile := []byte(`
+target "default" {
+}
+`)
+ dir := tmpdir(
+ t,
+ fstest.CreateFile("docker-bake.hcl", bakefile, 0600),
+ fstest.CreateFile("Dockerfile", dockerfile, 0600),
+ fstest.CreateFile("foo", []byte("foo"), 0600),
+ )
+
+ outputs := []string{
+ "--set", fmt.Sprintf("*.output=type=image,name=%s,push=true", targetReg),
+ "--set", fmt.Sprintf("*.output=type=docker,name=%s", targetStore),
+ "--set", fmt.Sprintf("*.output=type=oci,dest=%s/result", dir),
+ }
+ cmd := buildxCmd(sb, withDir(dir), withArgs("bake"), withArgs(outputs...))
+ cmd.Env = append(cmd.Env, "BUILDX_BUILDER="+builderName)
+ outb, err := cmd.CombinedOutput()
+ require.NoError(t, err, string(outb))
+
+ // test registry
+ desc, provider, err := contentutil.ProviderFromRef(targetReg)
+ require.NoError(t, err)
+ _, err = testutil.ReadImages(sb.Context(), provider, desc)
+ require.NoError(t, err)
+
+ // test docker store
+ cmd = dockerCmd(sb, withArgs("image", "inspect", targetStore))
+ cmd.Stderr = os.Stderr
+ require.NoError(t, cmd.Run())
+
+ // test oci
+ _, err = os.ReadFile(fmt.Sprintf("%s/result", dir))
+ require.NoError(t, err)
+
+ // TODO: test metadata file when supported by multi exporters https:/docker/buildx/issues/2181
+}
+
+func testBakeLoadPush(t *testing.T, sb integration.Sandbox) {
+ if sb.Name() != "docker" {
+ t.Skip("skipping test for non-docker workers")
+ }
+
+ registry, err := sb.NewRegistry()
+ if errors.Is(err, integration.ErrRequirements) {
+ t.Skip(err.Error())
+ }
+ require.NoError(t, err)
+
+ target := registry + "/buildx/registry:" + identity.NewID()
+
+ var builderName string
+ t.Cleanup(func() {
+ if builderName == "" {
+ return
+ }
+
+ cmd := dockerCmd(sb, withArgs("image", "rm", target))
+ cmd.Stderr = os.Stderr
+ require.NoError(t, cmd.Run())
+
+ out, err := rmCmd(sb, withArgs(builderName))
+ require.NoError(t, err, out)
+ })
+
+ // TODO: use stable buildkit image when v0.13.0 released
+ out, err := createCmd(sb, withArgs(
+ "--driver", "docker-container",
+ "--buildkitd-flags=--allow-insecure-entitlement=network.host",
+ "--driver-opt", "network=host",
+ "--driver-opt", "image=moby/buildkit:v0.13.0-rc3",
+ ))
+ require.NoError(t, err, out)
+ builderName = strings.TrimSpace(out)
+
+ dockerfile := []byte(`
+FROM scratch
+COPY foo /foo
+ `)
+ bakefile := []byte(`
+target "default" {
+}
+`)
+ dir := tmpdir(
+ t,
+ fstest.CreateFile("docker-bake.hcl", bakefile, 0600),
+ fstest.CreateFile("Dockerfile", dockerfile, 0600),
+ fstest.CreateFile("foo", []byte("foo"), 0600),
+ )
+
+ cmd := buildxCmd(sb, withDir(dir), withArgs("bake", "--push", "--load", fmt.Sprintf("--set=*.tags=%s", target)))
+ cmd.Env = append(cmd.Env, "BUILDX_BUILDER="+builderName)
+ outb, err := cmd.CombinedOutput()
+ require.NoError(t, err, string(outb))
+
+ // test registry
+ desc, provider, err := contentutil.ProviderFromRef(target)
+ require.NoError(t, err)
+ _, err = testutil.ReadImages(sb.Context(), provider, desc)
+ require.NoError(t, err)
+
+ // test docker store
+ cmd = dockerCmd(sb, withArgs("image", "inspect", target))
+ cmd.Stderr = os.Stderr
+ require.NoError(t, cmd.Run())
+
+ // TODO: test metadata file when supported by multi exporters https:/docker/buildx/issues/2181
+}