Skip to content

Commit

Permalink
dockerfile2llb: filter unused paths for named contexts
Browse files Browse the repository at this point in the history 
This changes the llb generation in `dockerfile2llb` to add the
`llb.FollowPaths` option when named contexts are used in the same way
that happens with the default context.

This means that dockerfiles that look like this:

```
FROM scratch
COPY --from=mynamedctx ./a.txt /a.txt
```

Will only load the file `a.txt`. This behavior is consistent with the
default context.

This also removes the unused []llb.LocalOption from ContextOpt and
updates the `NamedContext` function to return a `NamedContext` struct.
The `NamedContext` struct can be used to add additional local filtering
options onto the named context.

Signed-off-by: Jonathan A. Sternberg <[email protected]>
  • Loading branch information
@jsternberg
jsternberg committed Sep 20, 2023
1 parent bbe48e7 commit 250f926
Show file tree
Hide file tree
Showing 6 changed files with 299 additions and 97 deletions.
143 changes: 102 additions & 41 deletions frontend/dockerfile/dockerfile2llb/convert.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,21 +140,17 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
return nil, errors.Errorf("the Dockerfile cannot be empty")
}

namedContext := func(ctx context.Context, name string, copt dockerui.ContextOpt) (*llb.State, *image.Image, error) {
namedContext := func(ctx context.Context, name string, copt dockerui.ContextOpt) (*dockerui.NamedContext, error) {
if opt.Client == nil {
return nil, nil, nil
return nil, nil
}
if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") {
if copt.Platform == nil {
copt.Platform = opt.TargetPlatform
}
st, img, err := opt.Client.NamedContext(ctx, name, copt)
if err != nil {
return nil, nil, err
}
return st, img, nil
return opt.Client.NamedContext(ctx, name, copt)
}
return nil, nil, nil
return nil, nil
}

if opt.Warn == nil {
Expand Down Expand Up @@ -217,6 +213,9 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS

allDispatchStates := newDispatchStates()

// keep track of which dispatch state is associated with each named context.
namedContexts := make(map[*dispatchState]*dockerui.NamedContext)

// set base state for every image
for i, st := range stages {
name, used, err := shlex.ProcessWordWithMatches(st.BaseName, metaArgsToMap(optMetaArgs))
Expand All @@ -232,6 +231,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
stage: st,
deps: make(map[*dispatchState]struct{}),
ctxPaths: make(map[string]struct{}),
paths: make(map[string]struct{}),
stageName: st.Name,
prefixPlatform: opt.MultiPlatformRequested,
outline: outline.clone(),
Expand All @@ -255,20 +255,21 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
}

if st.Name != "" {
s, img, err := namedContext(ctx, st.Name, dockerui.ContextOpt{Platform: ds.platform, ResolveMode: opt.ImageResolveMode.String()})
nc, err := namedContext(ctx, st.Name, dockerui.ContextOpt{Platform: ds.platform, ResolveMode: opt.ImageResolveMode.String()})
if err != nil {
return nil, err
}
if s != nil {
if nc != nil {
ds.noinit = true
ds.state = *s
if img != nil {
ds.image = clampTimes(*img, opt.Epoch)
if img.Architecture != "" && img.OS != "" {
ds.state = nc.State
namedContexts[ds] = nc
if nc.Image != nil {
ds.image = clampTimes(*nc.Image, opt.Epoch)
if nc.Image.Architecture != "" && nc.Image.OS != "" {
ds.platform = &ocispecs.Platform{
OS: img.OS,
Architecture: img.Architecture,
Variant: img.Variant,
OS: nc.Image.OS,
Architecture: nc.Image.Architecture,
Variant: nc.Image.Variant,
}
}
}
Expand Down Expand Up @@ -382,18 +383,19 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
d.stage.BaseName = reference.TagNameOnly(ref).String()

var isScratch bool
st, img, err := namedContext(ctx, d.stage.BaseName, dockerui.ContextOpt{ResolveMode: opt.ImageResolveMode.String(), Platform: platform})
nc, err := namedContext(ctx, d.stage.BaseName, dockerui.ContextOpt{ResolveMode: opt.ImageResolveMode.String(), Platform: platform})
if err != nil {
return err
}
if st != nil {
if img != nil {
d.image = *img
if nc != nil {
if nc.Image != nil {
d.image = *nc.Image
} else {
d.image = emptyImage(platformOpt.targetPlatform)
}
d.state = st.Platform(*platform)
d.state = nc.State.Platform(*platform)
d.platform = platform
namedContexts[d] = nc
return nil
}
if reachable {
Expand Down Expand Up @@ -485,9 +487,18 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
d.image = clone(d.base.image)
}

// Ensure platform is set.
if d.platform == nil {
d.platform = &d.opt.targetPlatform
}

// make sure that PATH is always set
if _, ok := shell.BuildEnvs(d.image.Config.Env)["PATH"]; !ok {
d.image.Config.Env = append(d.image.Config.Env, "PATH="+system.DefaultPathEnv(d.platform.OS))
var os string
if d.platform != nil {
os = d.platform.OS
}
d.image.Config.Env = append(d.image.Config.Env, "PATH="+system.DefaultPathEnv(os))
}

// initialize base metadata from image conf
Expand Down Expand Up @@ -563,10 +574,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
target.image.Config.Labels[k] = v
}

opts := []llb.LocalOption{}
if includePatterns := normalizeContextPaths(ctxPaths); includePatterns != nil {
opts = append(opts, llb.FollowPaths(includePatterns))
}
opts := filterPaths(ctxPaths)
if opt.Client != nil {
bctx, err := opt.Client.MainContext(ctx, opts...)
if err != nil {
Expand All @@ -578,6 +586,16 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
buildContext.Output = bctx.Output()
}

// Propagate the list of used paths from the dispatchState to the associated
// NamedContext. This list of paths is used to filter any local context if possible.
for ds, nc := range namedContexts {
if len(ds.paths) == 0 {
continue
}
opts := filterPaths(ds.paths)
nc.SetLocalOpts(opts)
}

defaults := []llb.ConstraintsOpt{
llb.Platform(platformOpt.targetPlatform),
}
Expand Down Expand Up @@ -617,6 +635,7 @@ func toCommand(ic instructions.Command, allDispatchStates *dispatchStates) (comm
stn = &dispatchState{
stage: instructions.Stage{BaseName: c.From, Location: ic.Location()},
deps: make(map[*dispatchState]struct{}),
paths: make(map[string]struct{}),
unregistered: true,
}
}
Expand Down Expand Up @@ -759,9 +778,19 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
location: c.Location(),
opt: opt,
})
if err == nil && len(cmd.sources) == 0 {
for _, src := range c.SourcePaths {
d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{}
if err == nil {
if len(cmd.sources) == 0 {
for _, src := range c.SourcePaths {
d.ctxPaths[path.Join("/", filepath.ToSlash(src))] = struct{}{}
}
} else {
source := cmd.sources[0]
if source.paths == nil {
source.paths = make(map[string]struct{})
}
for _, src := range c.SourcePaths {
source.paths[path.Join("/", filepath.ToSlash(src))] = struct{}{}
}
}
}
default:
Expand All @@ -770,17 +799,20 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
}

type dispatchState struct {
opt dispatchOpt
state llb.State
image image.Image
platform *ocispecs.Platform
stage instructions.Stage
base *dispatchState
noinit bool
deps map[*dispatchState]struct{}
buildArgs []instructions.KeyValuePairOptional
commands []command
ctxPaths map[string]struct{}
opt dispatchOpt
state llb.State
image image.Image
platform *ocispecs.Platform
stage instructions.Stage
base *dispatchState
noinit bool
deps map[*dispatchState]struct{}
buildArgs []instructions.KeyValuePairOptional
commands []command
// ctxPaths marks the paths this dispatchState uses from the build context.
ctxPaths map[string]struct{}
// paths marks the paths that are used by this dispatchState.
paths map[string]struct{}
ignoreCache bool
cmdSet bool
unregistered bool
Expand Down Expand Up @@ -809,12 +841,24 @@ func (dss *dispatchStates) addState(ds *dispatchState) {
if d, ok := dss.statesByName[ds.stage.BaseName]; ok {
ds.base = d
ds.outline = d.outline.clone()
// Unify the set of paths so both this state and the parent state
// refer to the same map when recording path usage.
unifyPaths(&ds.paths, d.paths)
}
if ds.stage.Name != "" {
dss.statesByName[strings.ToLower(ds.stage.Name)] = ds
}
}

// unifyPaths ensures src and target contain the union of their paths and
// both maps refer to the same map so updates are seen by both.
func unifyPaths(src *map[string]struct{}, target map[string]struct{}) {
for p := range *src {
target[p] = struct{}{}
}
*src = target
}

func (dss *dispatchStates) findStateByName(name string) (*dispatchState, bool) {
ds, ok := dss.statesByName[strings.ToLower(name)]
return ds, ok
Expand Down Expand Up @@ -876,6 +920,9 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE

customname := c.String()

// Run command can potentially access any file. Mark the full filesystem as used.
d.paths["/"] = struct{}{}

var args []string = c.CmdLine
if len(c.Files) > 0 {
if len(args) != 1 || !c.PrependShell {
Expand Down Expand Up @@ -1580,6 +1627,11 @@ func hasCircularDependency(states []*dispatchState) (bool, *dispatchState) {
}

func normalizeContextPaths(paths map[string]struct{}) []string {
// Avoid a useless allocation if the set of paths is empty.
if len(paths) == 0 {
return nil
}

pathSlice := make([]string, 0, len(paths))
for p := range paths {
if p == "/" {
Expand All @@ -1594,6 +1646,15 @@ func normalizeContextPaths(paths map[string]struct{}) []string {
return pathSlice
}

// filterPaths returns the local options required to filter an llb.Local
// to only the required paths.
func filterPaths(paths map[string]struct{}) []llb.LocalOption {
if includePaths := normalizeContextPaths(paths); len(includePaths) > 0 {
return []llb.LocalOption{llb.FollowPaths(includePaths)}
}
return nil
}

func proxyEnvFromBuildArgs(args map[string]string) *llb.ProxyEnv {
pe := &llb.ProxyEnv{}
isNil := true
Expand Down
4 changes: 4 additions & 0 deletions frontend/dockerfile/dockerfile2llb/convert_runmount.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool {
stn = &dispatchState{
stage: instructions.Stage{BaseName: from},
deps: make(map[*dispatchState]struct{}),
paths: make(map[string]struct{}),
unregistered: true,
}
}
Expand Down Expand Up @@ -136,6 +137,9 @@ func dispatchRunMounts(d *dispatchState, c *instructions.RunCommand, sources []*

if mount.From == "" {
d.ctxPaths[path.Join("/", filepath.ToSlash(mount.Source))] = struct{}{}
} else {
source := sources[i]
source.paths[path.Join("/", filepath.ToSlash(mount.Source))] = struct{}{}
}
}
return out, nil
Expand Down
Loading

0 comments on commit 250f926

Please sign in to comment.