diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index b507a1a0fe121..0568c025197a7 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -140,21 +140,17 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS return nil, errors.Errorf("the Dockerfile cannot be empty") } - namedContext := func(ctx context.Context, name string, copt dockerui.ContextOpt) (*llb.State, *image.Image, error) { + namedContext := func(ctx context.Context, name string, copt dockerui.ContextOpt) (*dockerui.NamedContext, error) { if opt.Client == nil { - return nil, nil, nil + return nil, nil } if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") { if copt.Platform == nil { copt.Platform = opt.TargetPlatform } - st, img, err := opt.Client.NamedContext(ctx, name, copt) - if err != nil { - return nil, nil, err - } - return st, img, nil + return opt.Client.NamedContext(ctx, name, copt) } - return nil, nil, nil + return nil, nil } if opt.Warn == nil { @@ -217,6 +213,9 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS allDispatchStates := newDispatchStates() + // keep track of which dispatch state is associated with each named context. + namedContexts := make(map[*dispatchState]*dockerui.NamedContext) + // set base state for every image for i, st := range stages { name, used, err := shlex.ProcessWordWithMatches(st.BaseName, metaArgsToMap(optMetaArgs)) @@ -232,6 +231,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS stage: st, deps: make(map[*dispatchState]struct{}), ctxPaths: make(map[string]struct{}), + paths: make(map[string]struct{}), stageName: st.Name, prefixPlatform: opt.MultiPlatformRequested, outline: outline.clone(), @@ -255,20 +255,21 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS } if st.Name != "" { - s, img, err := namedContext(ctx, st.Name, dockerui.ContextOpt{Platform: ds.platform, ResolveMode: opt.ImageResolveMode.String()}) + nc, err := namedContext(ctx, st.Name, dockerui.ContextOpt{Platform: ds.platform, ResolveMode: opt.ImageResolveMode.String()}) if err != nil { return nil, err } - if s != nil { + if nc != nil { ds.noinit = true - ds.state = *s - if img != nil { - ds.image = clampTimes(*img, opt.Epoch) - if img.Architecture != "" && img.OS != "" { + ds.state = nc.State + namedContexts[ds] = nc + if nc.Image != nil { + ds.image = clampTimes(*nc.Image, opt.Epoch) + if nc.Image.Architecture != "" && nc.Image.OS != "" { ds.platform = &ocispecs.Platform{ - OS: img.OS, - Architecture: img.Architecture, - Variant: img.Variant, + OS: nc.Image.OS, + Architecture: nc.Image.Architecture, + Variant: nc.Image.Variant, } } } @@ -382,18 +383,19 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS d.stage.BaseName = reference.TagNameOnly(ref).String() var isScratch bool - st, img, err := namedContext(ctx, d.stage.BaseName, dockerui.ContextOpt{ResolveMode: opt.ImageResolveMode.String(), Platform: platform}) + nc, err := namedContext(ctx, d.stage.BaseName, dockerui.ContextOpt{ResolveMode: opt.ImageResolveMode.String(), Platform: platform}) if err != nil { return err } - if st != nil { - if img != nil { - d.image = *img + if nc != nil { + if nc.Image != nil { + d.image = *nc.Image } else { d.image = emptyImage(platformOpt.targetPlatform) } - d.state = st.Platform(*platform) + d.state = nc.State.Platform(*platform) d.platform = platform + namedContexts[d] = nc return nil } if reachable { @@ -485,9 +487,18 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS d.image = clone(d.base.image) } + // Ensure platform is set. + if d.platform == nil { + d.platform = &d.opt.targetPlatform + } + // make sure that PATH is always set if _, ok := shell.BuildEnvs(d.image.Config.Env)["PATH"]; !ok { - d.image.Config.Env = append(d.image.Config.Env, "PATH="+system.DefaultPathEnv(d.platform.OS)) + var os string + if d.platform != nil { + os = d.platform.OS + } + d.image.Config.Env = append(d.image.Config.Env, "PATH="+system.DefaultPathEnv(os)) } // initialize base metadata from image conf @@ -563,10 +574,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS target.image.Config.Labels[k] = v } - opts := []llb.LocalOption{} - if includePatterns := normalizeContextPaths(ctxPaths); includePatterns != nil { - opts = append(opts, llb.FollowPaths(includePatterns)) - } + opts := filterPaths(ctxPaths) if opt.Client != nil { bctx, err := opt.Client.MainContext(ctx, opts...) if err != nil { @@ -578,6 +586,16 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS buildContext.Output = bctx.Output() } + // Propagate the list of used paths from the dispatchState to the associated + // NamedContext. This list of paths is used to filter any local context if possible. + for ds, nc := range namedContexts { + if len(ds.paths) == 0 { + continue + } + opts := filterPaths(ds.paths) + nc.SetLocalOpts(opts) + } + defaults := []llb.ConstraintsOpt{ llb.Platform(platformOpt.targetPlatform), } @@ -617,6 +635,7 @@ func toCommand(ic instructions.Command, allDispatchStates *dispatchStates) (comm stn = &dispatchState{ stage: instructions.Stage{BaseName: c.From, Location: ic.Location()}, deps: make(map[*dispatchState]struct{}), + paths: make(map[string]struct{}), unregistered: true, } } @@ -759,9 +778,19 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error { location: c.Location(), opt: opt, }) - if err == nil && len(cmd.sources) == 0 { - for _, src := range c.SourcePaths { - d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{} + if err == nil { + if len(cmd.sources) == 0 { + for _, src := range c.SourcePaths { + d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{} + } + } else { + source := cmd.sources[0] + if source.paths == nil { + source.paths = make(map[string]struct{}) + } + for _, src := range c.SourcePaths { + source.paths[path.Join("/", filepath.ToSlash(src))] = struct{}{} + } } } default: @@ -770,17 +799,20 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error { } type dispatchState struct { - opt dispatchOpt - state llb.State - image image.Image - platform *ocispecs.Platform - stage instructions.Stage - base *dispatchState - noinit bool - deps map[*dispatchState]struct{} - buildArgs []instructions.KeyValuePairOptional - commands []command - ctxPaths map[string]struct{} + opt dispatchOpt + state llb.State + image image.Image + platform *ocispecs.Platform + stage instructions.Stage + base *dispatchState + noinit bool + deps map[*dispatchState]struct{} + buildArgs []instructions.KeyValuePairOptional + commands []command + // ctxPaths marks the paths this dispatchState uses from the build context. + ctxPaths map[string]struct{} + // paths marks the paths that are used by this dispatchState. + paths map[string]struct{} ignoreCache bool cmdSet bool unregistered bool @@ -809,12 +841,24 @@ func (dss *dispatchStates) addState(ds *dispatchState) { if d, ok := dss.statesByName[ds.stage.BaseName]; ok { ds.base = d ds.outline = d.outline.clone() + // Unify the set of paths so both this state and the parent state + // refer to the same map when recording path usage. + unifyPaths(&ds.paths, d.paths) } if ds.stage.Name != "" { dss.statesByName[strings.ToLower(ds.stage.Name)] = ds } } +// unifyPaths ensures src and target contain the union of their paths and +// both maps refer to the same map so updates are seen by both. +func unifyPaths(src *map[string]struct{}, target map[string]struct{}) { + for p := range *src { + target[p] = struct{}{} + } + *src = target +} + func (dss *dispatchStates) findStateByName(name string) (*dispatchState, bool) { ds, ok := dss.statesByName[strings.ToLower(name)] return ds, ok @@ -876,6 +920,9 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE customname := c.String() + // Run command can potentially access any file. Mark the full filesystem as used. + d.paths["/"] = struct{}{} + var args []string = c.CmdLine if len(c.Files) > 0 { if len(args) != 1 || !c.PrependShell { @@ -1580,6 +1627,11 @@ func hasCircularDependency(states []*dispatchState) (bool, *dispatchState) { } func normalizeContextPaths(paths map[string]struct{}) []string { + // Avoid a useless allocation if the set of paths is empty. + if len(paths) == 0 { + return nil + } + pathSlice := make([]string, 0, len(paths)) for p := range paths { if p == "/" { @@ -1594,6 +1646,15 @@ func normalizeContextPaths(paths map[string]struct{}) []string { return pathSlice } +// filterPaths returns the local options required to filter an llb.Local +// to only the required paths. +func filterPaths(paths map[string]struct{}) []llb.LocalOption { + if includePaths := normalizeContextPaths(paths); len(includePaths) > 0 { + return []llb.LocalOption{llb.FollowPaths(includePaths)} + } + return nil +} + func proxyEnvFromBuildArgs(args map[string]string) *llb.ProxyEnv { pe := &llb.ProxyEnv{} isNil := true diff --git a/frontend/dockerfile/dockerfile2llb/convert_runmount.go b/frontend/dockerfile/dockerfile2llb/convert_runmount.go index 7485357babcd9..bac9b948a4e1a 100644 --- a/frontend/dockerfile/dockerfile2llb/convert_runmount.go +++ b/frontend/dockerfile/dockerfile2llb/convert_runmount.go @@ -31,6 +31,7 @@ func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool { stn = &dispatchState{ stage: instructions.Stage{BaseName: from}, deps: make(map[*dispatchState]struct{}), + paths: make(map[string]struct{}), unregistered: true, } } @@ -136,6 +137,9 @@ func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []* if mount.From == "" { d.ctxPaths[path.Join("/", filepath.ToSlash(mount.Source))] = struct{}{} + } else { + source := sources[i] + source.paths[path.Join("/", filepath.ToSlash(mount.Source))] = struct{}{} } } return out, nil diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 4bb64914045c3..6b0eef8852aab 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -8,12 +8,14 @@ import ( "encoding/json" "fmt" "io" + "math" "net/http" "net/http/httptest" "os" "os/exec" "path" "path/filepath" + "regexp" "runtime" "sort" "strings" @@ -21,6 +23,7 @@ import ( "time" v1 "github.com/moby/buildkit/cache/remotecache/v1" + "golang.org/x/sync/errgroup" "github.com/containerd/containerd" "github.com/containerd/containerd/content" @@ -130,6 +133,7 @@ var allTests = integration.TestFuncs( testNamedOCILayoutContextExport, testNamedInputContext, testNamedMultiplatformInputContext, + testNamedFilteredContext, testEmptyDestDir, testCopyChownCreateDest, testCopyThroughSymlinkContext, @@ -6063,6 +6067,116 @@ COPY --from=build /foo /out / require.Equal(t, "foo is bar-arm64\n", string(dt)) } +func testNamedFilteredContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dir2 := integration.Tmpdir(t, + // small file + fstest.CreateFile("foo", []byte(`foo`), 0600), + // blank file that's just large + fstest.CreateFile("bar", make([]byte, 4096*1000), 0600), + ) + + f := getFrontend(t, sb) + + runTest := func(t *testing.T, dockerfile []byte, target string, min, max int64) { + t.Run(target, func(t *testing.T) { + dir := integration.Tmpdir( + t, + fstest.CreateFile(dockerui.DefaultDockerfileName, dockerfile, 0600), + ) + + ch := make(chan *client.SolveStatus) + + eg, ctx := errgroup.WithContext(sb.Context()) + eg.Go(func() error { + _, err := f.Solve(ctx, c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context:foo": "local:foo", + "target": target, + }, + LocalDirs: map[string]string{ + dockerui.DefaultLocalNameDockerfile: dir, + dockerui.DefaultLocalNameContext: dir, + "foo": dir2, + }, + }, ch) + return err + }) + + eg.Go(func() error { + transferred := make(map[string]int64) + re := regexp.MustCompile(`transferring (.+):`) + for ss := range ch { + for _, status := range ss.Statuses { + m := re.FindStringSubmatch(status.ID) + if m == nil { + continue + } + + ctxName := m[1] + t.Logf("%s %d", ctxName, status.Current) + transferred[ctxName] = status.Current + } + } + + t.Logf("transferred %+v", transferred) + if foo := transferred["foo"]; foo < min { + return errors.Errorf("not enough data was transferred, %d < %d", foo, min) + } else if foo > max { + return errors.Errorf("too much data was transferred, %d > %d", foo, max) + } + return nil + }) + + err := eg.Wait() + require.NoError(t, err) + }) + } + + dockerfileBase := []byte(` +FROM scratch AS copy_from +COPY --from=foo /foo / + +FROM alpine AS run_mount +RUN --mount=from=foo,src=/foo,target=/in/foo cp /in/foo /foo + +FROM foo AS image_source +COPY --from=alpine / / +RUN cat /foo > /bar + +FROM scratch AS all +COPY --link --from=copy_from /foo /foo.b +COPY --link --from=run_mount /foo /foo.c +COPY --link --from=image_source /bar /foo.d +`) + + t.Run("new", func(t *testing.T) { + runTest(t, dockerfileBase, "run_mount", 1, 1024) + runTest(t, dockerfileBase, "copy_from", 1, 1024) + runTest(t, dockerfileBase, "image_source", 4096*1000, math.MaxInt64) + runTest(t, dockerfileBase, "all", 4096*1000, math.MaxInt64) + }) + + dockerfileFull := append([]byte(` +FROM scratch AS foo +COPY < maxContextRecursion { - return nil, nil, errors.New("context recursion limit exceeded; this may indicate a cycle in the provided source policies: " + v) + return nil, errors.New("context recursion limit exceeded; this may indicate a cycle in the provided source policies: " + v) } vv := strings.SplitN(v, ":", 2) if len(vv) != 2 { - return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, nameWithPlatform) + return nil, errors.Errorf("invalid context specifier %s for %s", v, nameWithPlatform) } // allow git@ without protocol for SSH URLs for backwards compatibility @@ -52,7 +52,7 @@ func (bc *Client) namedContextRecursive(ctx context.Context, name string, nameWi ref := strings.TrimPrefix(vv[1], "//") if ref == EmptyImageName { st := llb.Scratch() - return &st, nil, nil + return &NamedContext{State: st}, nil } imgOpt := []llb.ImageOption{ @@ -64,7 +64,7 @@ func (bc *Client) namedContextRecursive(ctx context.Context, name string, nameWi named, err := reference.ParseNormalizedNamed(ref) if err != nil { - return nil, nil, err + return nil, err } named = reference.TagNameOnly(named) @@ -80,61 +80,64 @@ func (bc *Client) namedContextRecursive(ctx context.Context, name string, nameWi if errors.As(err, &e) { return bc.namedContextRecursive(ctx, e.Updated, name, opt, count+1) } - return nil, nil, err + return nil, err } var img image.Image if err := json.Unmarshal(data, &img); err != nil { - return nil, nil, err + return nil, err } img.Created = nil st := llb.Image(ref, imgOpt...) st, err = st.WithImageConfig(data) if err != nil { - return nil, nil, err + return nil, err } if opt.CaptureDigest != nil { *opt.CaptureDigest = dgst } - return &st, &img, nil + return &NamedContext{ + State: st, + Image: &img, + }, nil case "git": st, ok := DetectGitContext(v, true) if !ok { - return nil, nil, errors.Errorf("invalid git context %s", v) + return nil, errors.Errorf("invalid git context %s", v) } - return st, nil, nil + return &NamedContext{State: *st}, nil case "http", "https": st, ok := DetectGitContext(v, true) if !ok { httpst := llb.HTTP(v, llb.WithCustomName("[context "+nameWithPlatform+"] "+v)) st = &httpst } - return st, nil, nil + return &NamedContext{State: *st}, nil case "oci-layout": refSpec := strings.TrimPrefix(vv[1], "//") ref, err := reference.Parse(refSpec) if err != nil { - return nil, nil, errors.Wrapf(err, "could not parse oci-layout reference %q", refSpec) + return nil, errors.Wrapf(err, "could not parse oci-layout reference %q", refSpec) } named, ok := ref.(reference.Named) if !ok { - return nil, nil, errors.Errorf("oci-layout reference %q has no name", ref.String()) + return nil, errors.Errorf("oci-layout reference %q has no name", ref.String()) } dgstd, ok := named.(reference.Digested) if !ok { - return nil, nil, errors.Errorf("oci-layout reference %q has no digest", named.String()) + return nil, errors.Errorf("oci-layout reference %q has no digest", named.String()) } // for the dummy ref primarily used in log messages, we can use the // original name, since the store key may not be significant dummyRef, err := reference.ParseNormalizedNamed(name) if err != nil { - return nil, nil, errors.Wrapf(err, "could not parse oci-layout reference %q", name) + return nil, errors.Wrapf(err, "could not parse oci-layout reference %q", name) } dummyRef, err = reference.WithDigest(dummyRef, dgstd.Digest()) if err != nil { - return nil, nil, errors.Wrapf(err, "could not wrap %q with digest", name) + return nil, errors.Wrapf(err, "could not wrap %q with digest", name) } // TODO: How should source policy be handled here with a dummy ref? @@ -149,12 +152,12 @@ func (bc *Client) namedContextRecursive(ctx context.Context, name string, nameWi }, }) if err != nil { - return nil, nil, err + return nil, err } var img image.Image if err := json.Unmarshal(data, &img); err != nil { - return nil, nil, errors.Wrap(err, "could not parse oci-layout image config") + return nil, errors.Wrap(err, "could not parse oci-layout image config") } ociOpt := []llb.OCILayoutOption{ @@ -170,12 +173,15 @@ func (bc *Client) namedContextRecursive(ctx context.Context, name string, nameWi ) st, err = st.WithImageConfig(data) if err != nil { - return nil, nil, err + return nil, err } if opt.CaptureDigest != nil { *opt.CaptureDigest = dgst } - return &st, &img, nil + return &NamedContext{ + State: st, + Image: &img, + }, nil case "local": st := llb.Local(vv[1], llb.SessionID(bc.bopts.SessionID), @@ -186,18 +192,18 @@ func (bc *Client) namedContextRecursive(ctx context.Context, name string, nameWi ) def, err := st.Marshal(ctx) if err != nil { - return nil, nil, err + return nil, err } res, err := bc.client.Solve(ctx, client.SolveRequest{ Evaluate: true, Definition: def.ToPB(), }) if err != nil { - return nil, nil, err + return nil, err } ref, err := res.SingleRef() if err != nil { - return nil, nil, err + return nil, err } var excludes []string if !opt.NoDockerignore { @@ -208,46 +214,58 @@ func (bc *Client) namedContextRecursive(ctx context.Context, name string, nameWi if len(dt) != 0 { excludes, err = ignorefile.ReadAll(bytes.NewBuffer(dt)) if err != nil { - return nil, nil, errors.Wrapf(err, "failed parsing %s", DefaultDockerignoreName) + return nil, errors.Wrapf(err, "failed parsing %s", DefaultDockerignoreName) } } } - st = llb.Local(vv[1], - llb.WithCustomName("[context "+nameWithPlatform+"] load from client"), - llb.SessionID(bc.bopts.SessionID), - llb.SharedKeyHint("context:"+nameWithPlatform), - llb.ExcludePatterns(excludes), - ) - return &st, nil, nil + + // Wrap the llb.Local call in Async to allow modification to the local opts filter + // before marshaling the request. + nc := &NamedContext{} + nc.State = llb.Scratch().Async(func(ctx context.Context, _ llb.State, _ *llb.Constraints) (llb.State, error) { + opts := append([]llb.LocalOption{ + llb.WithCustomName("[context " + nameWithPlatform + "] load from client"), + llb.SessionID(bc.bopts.SessionID), + llb.SharedKeyHint("context:" + nameWithPlatform), + llb.ExcludePatterns(excludes), + }, nc.localOpts...) + return llb.Local(vv[1], opts...), nil + }) + return nc, nil case "input": inputs, err := bc.client.Inputs(ctx) if err != nil { - return nil, nil, err + return nil, err } st, ok := inputs[vv[1]] if !ok { - return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], nameWithPlatform) + return nil, errors.Errorf("invalid input %s for %s", vv[1], nameWithPlatform) } md, ok := opts[inputMetadataPrefix+vv[1]] if ok { m := make(map[string][]byte) if err := json.Unmarshal([]byte(md), &m); err != nil { - return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) + return nil, errors.Wrapf(err, "failed to parse input metadata %s", md) } var img *image.Image if dtic, ok := m[exptypes.ExporterImageConfigKey]; ok { st, err = st.WithImageConfig(dtic) if err != nil { - return nil, nil, err + return nil, err } if err := json.Unmarshal(dtic, &img); err != nil { - return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", nameWithPlatform) + return nil, errors.Wrapf(err, "failed to parse image config for %s", nameWithPlatform) } } - return &st, img, nil + return &NamedContext{ + State: st, + Image: img, + }, nil } - return &st, nil, nil + return &NamedContext{ + State: st, + }, nil default: - return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], nameWithPlatform) + return nil, errors.Errorf("unsupported context source %s for %s", vv[0], nameWithPlatform) } } diff --git a/frontend/gateway/gateway.go b/frontend/gateway/gateway.go index 3b23386fc8e4d..072599a161bd3 100644 --- a/frontend/gateway/gateway.go +++ b/frontend/gateway/gateway.go @@ -149,15 +149,12 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten if err != nil { return nil, err } - st, dockerImage, err := dc.NamedContext(ctx, source, dockerui.ContextOpt{ + st, err := dc.NamedContext(ctx, source, dockerui.ContextOpt{ CaptureDigest: &mfstDigest, }) if err != nil { return nil, err } - if dockerImage != nil { - img = *dockerImage - } if st == nil { sourceRef, err := reference.ParseNormalizedNamed(source) if err != nil { @@ -188,7 +185,9 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten } src := llb.Image(sourceRef.String(), &markTypeFrontend{}) - st = &src + st = &dockerui.NamedContext{State: src} + } else if st.Image != nil { + img = *st.Image } def, err := st.Marshal(ctx)