bake: add new --sync-output flag

This patch introduces a new syncable output option, which ensures that
all builds finish simultaneously in the solver, so that no outputs are
completed independently of each other. This allows bake to easily
express the notion that either all builds should succeed and output or
none of them should.

Signed-off-by: Justin Chadwell <[email protected]>

build/build.go

@@ -15,6 +15,7 @@ import (
  "strconv"
  "strings"
  "sync"
+ "sync/atomic"
  "syscall"
  "time"
 
@@ -737,11 +738,11 @@ func Invoke(ctx context.Context, cfg ContainerConfig) error {
  return err
 }
 
-func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer) (resp map[string]*client.SolveResponse, err error) {
- return BuildWithResultHandler(ctx, drivers, opt, docker, configDir, w, nil, false)
+func Build(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, syncOutputs bool) (resp map[string]*client.SolveResponse, err error) {
+ return BuildWithResultHandler(ctx, drivers, opt, docker, configDir, w, nil, syncOutputs, false)
 }
 
-func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext), allowNoOutput bool) (resp map[string]*client.SolveResponse, err error) {
+func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[string]Options, docker DockerAPI, configDir string, w progress.Writer, resultHandleFunc func(driverIndex int, rCtx *ResultContext), syncOutputs bool, allowNoOutput bool) (resp map[string]*client.SolveResponse, err error) {
  if len(drivers) == 0 {
  return nil, errors.Errorf("driver required for build")
  }
@@ -806,9 +807,11 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
  }
  }
 
+ msize := 0
  for k, opt := range opt {
  multiDriver := len(m[k]) > 1
  hasMobyDriver := false
+ msize += len(m[k])
  for i, dp := range m[k] {
  di := drivers[dp.driverIndex]
  if di.Driver.IsMobyDriver() {
@@ -880,6 +883,10 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
 
  multiTarget := len(opt) > 1
 
+ buildGrp := &sync.WaitGroup{}
+ buildGrp.Add(msize)
+ errCount := int64(0)
+
  for k, opt := range opt {
  err := func(k string) error {
  opt := opt
@@ -1068,6 +1075,7 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
  }
 
  req := gateway.SolveRequest{
+ Evaluate: syncOutputs,
  Frontend: so.Frontend,
  FrontendOpt: so.FrontendAttrs,
  FrontendInputs: frontendInputs,
@@ -1099,6 +1107,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
  res, err := c.Solve(ctx, req)
  if err != nil {
  if origErr != nil {
+ atomic.AddInt64(&errCount, 1)
+ buildGrp.Done()
  return nil, err
  }
  var reqErr *errdefs.UnsupportedSubrequestError
@@ -1110,6 +1120,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
  origErr = err
  continue
  }
+ atomic.AddInt64(&errCount, 1)
+ buildGrp.Done()
  return nil, err
  }
  // buildkit v0.8 vendored in Docker 20.10 does not support typed errors
@@ -1119,6 +1131,8 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
  continue
  }
  }
+ atomic.AddInt64(&errCount, 1)
+ buildGrp.Done()
  return nil, err
  }
  if opt.PrintFunc != nil {
@@ -1128,6 +1142,17 @@ func BuildWithResultHandler(ctx context.Context, drivers []DriverInfo, opt map[s
  if resultHandleFunc != nil {
  resultHandleFunc(dp.driverIndex, &ResultContext{cc, res})
  }
+
+ buildGrp.Done()
+ if syncOutputs {
+ buildGrp.Wait()
+ if atomic.LoadInt64(&errCount) > 0 {
+ // wait until cancelled
+ <-ctx.Done()
+ return nil, ctx.Err()
+ }
+ }
+
  return res, nil
  }
  }, ch)

commands/bake.go

@@ -19,9 +19,10 @@ import (
 )
 
 type bakeOptions struct {
- files []string
- overrides []string
- printOnly bool
+ files []string
+ overrides []string
+ printOnly bool
+ syncOutput bool
  commonOptions
 }
 
@@ -145,7 +146,7 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) (err error
  return nil
  }
 
- resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer)
+ resp, err := build.Build(ctx, dis, bo, dockerAPI(dockerCli), confutil.ConfigDir(dockerCli), printer, in.syncOutput)
  if err != nil {
  return wrapBuildError(err, true)
  }
@@ -189,6 +190,7 @@ func bakeCmd(dockerCli command.Cli, rootOpts *rootOptions) *cobra.Command {
  flags.BoolVar(&options.exportLoad, "load", false, `Shorthand for "--set=*.output=type=docker"`)
  flags.BoolVar(&options.printOnly, "print", false, "Print the options without building")
  flags.BoolVar(&options.exportPush, "push", false, `Shorthand for "--set=*.output=type=registry"`)
+ flags.BoolVar(&options.syncOutput, "sync-output", false, "Ensure all builds complete before beginning output")
  flags.StringArrayVar(&options.overrides, "set", nil, `Override target value (e.g., "targetpattern.key=value")`)
 
  commonBuildFlags(&options.commonOptions, flags)

commands/build.go

@@ -297,7 +297,7 @@ func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]bu
  if res == nil || driverIndex < idx {
  idx, res = driverIndex, gotRes
  }
- }, allowNoOutput)
+ }, false, allowNoOutput)
  err1 := printer.Wait()
  if err == nil {
  err = err1

docs/reference/buildx_bake.md

@@ -25,6 +25,7 @@ Build from a file
 | [`--pull`](#pull) | | | Always attempt to pull all referenced images |
 | `--push` | | | Shorthand for `--set=*.output=type=registry` |
 | [`--set`](#set) | `stringArray` | | Override target value (e.g., `targetpattern.key=value`) |
+| `--sync-output` | | | Ensure all builds complete before beginning output |
 
 
 <!---MARKER_GEN_END-->