Skip to content

Commit

Permalink
Merge pull request #4874 from tonistiigi/v0.13.2-picks
Browse files Browse the repository at this point in the history
[v0.13] cherry-picks for v0.13.2
  • Loading branch information
tonistiigi authored Apr 25, 2024
2 parents 4317777 + 25108ab commit 2e18d70
Show file tree
Hide file tree
Showing 19 changed files with 416 additions and 53 deletions.
52 changes: 52 additions & 0 deletions client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
testFileOpInputSwap,
testRelativeMountpoint,
testLocalSourceDiffer,
testLocalSourceWithHardlinksFilter,
testOCILayoutSource,
testOCILayoutPlatformSource,
testBuildExportZstd,
Expand Down Expand Up @@ -1964,6 +1965,57 @@ func testLocalSourceWithDiffer(t *testing.T, sb integration.Sandbox, d llb.DiffT
}
}

// moby/buildkit#4831
func testLocalSourceWithHardlinksFilter(t *testing.T, sb integration.Sandbox) {
requiresLinux(t)
c, err := New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()

dir := integration.Tmpdir(
t,
fstest.CreateFile("bar", []byte("bar"), 0600),
fstest.Link("bar", "foo1"),
fstest.Link("bar", "foo2"),
)

st := llb.Local("mylocal", llb.FollowPaths([]string{"foo*"}))

def, err := st.Marshal(context.TODO())
require.NoError(t, err)

destDir := t.TempDir()

_, err = c.Solve(context.TODO(), def, SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterLocal,
OutputDir: destDir,
},
},
LocalMounts: map[string]fsutil.FS{
"mylocal": dir,
},
}, nil)
require.NoError(t, err)

_, err = os.ReadFile(filepath.Join(destDir, "bar"))
require.Error(t, err)
require.True(t, os.IsNotExist(err))

dt, err := os.ReadFile(filepath.Join(destDir, "foo1"))
require.NoError(t, err)
require.Equal(t, []byte("bar"), dt)

st1, err := os.Stat(filepath.Join(destDir, "foo1"))
require.NoError(t, err)

st2, err := os.Stat(filepath.Join(destDir, "foo2"))
require.NoError(t, err)

require.True(t, os.SameFile(st1, st2))
}

func testOCILayoutSource(t *testing.T, sb integration.Sandbox) {
workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureOCILayout)
requiresLinux(t)
Expand Down
9 changes: 5 additions & 4 deletions cmd/buildkitd/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,11 @@ type LogConfig struct {
}

type GRPCConfig struct {
Address []string `toml:"address"`
DebugAddress string `toml:"debugAddress"`
UID *int `toml:"uid"`
GID *int `toml:"gid"`
Address []string `toml:"address"`
DebugAddress string `toml:"debugAddress"`
UID *int `toml:"uid"`
GID *int `toml:"gid"`
SecurityDescriptor string `toml:"securityDescriptor"`

TLS TLSConfig `toml:"tls"`
// MaxRecvMsgSize int `toml:"max_recv_message_size"`
Expand Down
23 changes: 23 additions & 0 deletions cmd/buildkitd/config/gcpolicy.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"time"

"github.com/docker/go-units"
"github.com/moby/buildkit/util/bklog"
"github.com/pkg/errors"
)

Expand Down Expand Up @@ -104,3 +105,25 @@ func stripQuotes(s string) string {
}
return s
}

func DetectDefaultGCCap() DiskSpace {
return DiskSpace{Percentage: DiskSpacePercentage}
}

func (d DiskSpace) AsBytes(root string) int64 {
if d.Bytes != 0 {
return d.Bytes
}
if d.Percentage == 0 {
return 0
}

diskSize, err := getDiskSize(root)
if err != nil {
bklog.L.Warnf("failed to get disk size: %v", err)
return defaultCap
}
avail := diskSize * d.Percentage / 100
rounded := (avail/(1<<30) + 1) * 1e9 // round up
return rounded
}
18 changes: 4 additions & 14 deletions cmd/buildkitd/config/gcpolicy_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,13 @@ import (
"syscall"
)

func DetectDefaultGCCap() DiskSpace {
return DiskSpace{Percentage: 10}
}

func (d DiskSpace) AsBytes(root string) int64 {
if d.Bytes != 0 {
return d.Bytes
}
if d.Percentage == 0 {
return 0
}
var DiskSpacePercentage int64 = 10

func getDiskSize(root string) (int64, error) {
var st syscall.Statfs_t
if err := syscall.Statfs(root, &st); err != nil {
return defaultCap
return 0, err
}
diskSize := int64(st.Bsize) * int64(st.Blocks)
avail := diskSize * d.Percentage / 100
return (avail/(1<<30) + 1) * 1e9 // round up
return diskSize, nil
}
29 changes: 24 additions & 5 deletions cmd/buildkitd/config/gcpolicy_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,29 @@

package config

func DetectDefaultGCCap() DiskSpace {
return DiskSpace{Bytes: defaultCap}
}
import (
"golang.org/x/sys/windows"
)

// set as double that for Linux since
// Windows images are generally larger.
var DiskSpacePercentage int64 = 20

func getDiskSize(root string) (int64, error) {
rootUTF16, err := windows.UTF16FromString(root)
if err != nil {
return 0, err
}
var freeAvailableBytes uint64
var totalBytes uint64
var totalFreeBytes uint64

func (d DiskSpace) AsBytes(root string) int64 {
return d.Bytes
if err := windows.GetDiskFreeSpaceEx(
&rootUTF16[0],
&freeAvailableBytes,
&totalBytes,
&totalFreeBytes); err != nil {
return 0, err
}
return int64(totalBytes), nil
}
35 changes: 28 additions & 7 deletions cmd/buildkitd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"os"
"os/user"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
Expand Down Expand Up @@ -397,9 +398,18 @@ func newGRPCListeners(cfg config.GRPCConfig) ([]net.Listener, error) {
if err != nil {
return nil, err
}

sd := cfg.SecurityDescriptor
if sd == "" {
sd, err = groupToSecurityDescriptor("")
if err != nil {
return nil, err
}
}

listeners := make([]net.Listener, 0, len(addrs))
for _, addr := range addrs {
l, err := getListener(addr, *cfg.UID, *cfg.GID, tlsConfig)
l, err := getListener(addr, *cfg.UID, *cfg.GID, sd, tlsConfig)
if err != nil {
for _, l := range listeners {
l.Close()
Expand Down Expand Up @@ -569,11 +579,19 @@ func applyMainFlags(c *cli.Context, cfg *config.Config) error {
}

if group := c.String("group"); group != "" {
gid, err := groupToGid(group)
if err != nil {
return err
if runtime.GOOS == "windows" {
secDescriptor, err := groupToSecurityDescriptor(group)
if err != nil {
return err
}
cfg.GRPC.SecurityDescriptor = secDescriptor
} else {
gid, err := groupToGid(group)
if err != nil {
return err
}
cfg.GRPC.GID = &gid
}
cfg.GRPC.GID = &gid
}

if tlscert := c.String("tlscert"); tlscert != "" {
Expand Down Expand Up @@ -628,7 +646,7 @@ func groupToGid(group string) (int, error) {
return id, nil
}

func getListener(addr string, uid, gid int, tlsConfig *tls.Config) (net.Listener, error) {
func getListener(addr string, uid, gid int, secDescriptor string, tlsConfig *tls.Config) (net.Listener, error) {
addrSlice := strings.SplitN(addr, "://", 2)
if len(addrSlice) < 2 {
return nil, errors.Errorf("address %s does not contain proto, you meant unix://%s ?",
Expand All @@ -641,6 +659,9 @@ func getListener(addr string, uid, gid int, tlsConfig *tls.Config) (net.Listener
if tlsConfig != nil {
bklog.L.Warnf("TLS is disabled for %s", addr)
}
if proto == "npipe" {
return getLocalListener(listenAddr, secDescriptor)
}
return sys.GetLocalListener(listenAddr, uid, gid)
case "fd":
return listenFD(listenAddr, tlsConfig)
Expand Down Expand Up @@ -928,7 +949,7 @@ func parseBoolOrAuto(s string) (*bool, error) {
func runTraceController(p string, exp sdktrace.SpanExporter) error {
server := grpc.NewServer()
tracev1.RegisterTraceServiceServer(server, &traceCollector{exporter: exp})
l, err := getLocalListener(p)
l, err := getLocalListener(p, "")
if err != nil {
return errors.Wrap(err, "creating trace controller listener")
}
Expand Down
6 changes: 5 additions & 1 deletion cmd/buildkitd/main_unix.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func listenFD(addr string, tlsConfig *tls.Config) (net.Listener, error) {
return nil, errors.New("not supported yet")
}

func getLocalListener(listenerPath string) (net.Listener, error) {
func getLocalListener(listenerPath, _ string) (net.Listener, error) {
uid := os.Getuid()
l, err := sys.GetLocalListener(listenerPath, uid, uid)
if err != nil {
Expand All @@ -60,3 +60,7 @@ func getLocalListener(listenerPath string) (net.Listener, error) {
}
return l, nil
}

func groupToSecurityDescriptor(_ string) (string, error) {
return "", nil
}
26 changes: 23 additions & 3 deletions cmd/buildkitd/main_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ package main

import (
"crypto/tls"
"fmt"
"net"
"strings"

"github.com/Microsoft/go-winio"
_ "github.com/moby/buildkit/solver/llbsolver/ops"
Expand All @@ -19,14 +21,18 @@ func listenFD(addr string, tlsConfig *tls.Config) (net.Listener, error) {
return nil, errors.New("listening server on fd not supported on windows")
}

func getLocalListener(listenerPath string) (net.Listener, error) {
pc := &winio.PipeConfig{
func getLocalListener(listenerPath, secDescriptor string) (net.Listener, error) {
if secDescriptor == "" {
// Allow generic read and generic write access to authenticated users
// and system users. On Linux, this pipe seems to be given rw access to
// user, group and others (666).
// TODO(gabriel-samfira): should we restrict access to this pipe to just
// authenticated users? Or Administrators group?
SecurityDescriptor: "D:P(A;;GRGW;;;AU)(A;;GRGW;;;SY)",
secDescriptor = "D:P(A;;GRGW;;;AU)(A;;GRGW;;;SY)"
}

pc := &winio.PipeConfig{
SecurityDescriptor: secDescriptor,
}

listener, err := winio.ListenPipe(listenerPath, pc)
Expand All @@ -35,3 +41,17 @@ func getLocalListener(listenerPath string) (net.Listener, error) {
}
return listener, nil
}

func groupToSecurityDescriptor(group string) (string, error) {
sddl := "D:P(A;;GA;;;BA)(A;;GA;;;SY)"
if group != "" {
for _, g := range strings.Split(group, ",") {
sid, err := winio.LookupSidByName(g)
if err != nil {
return "", errors.Wrapf(err, "failed to lookup sid for group %s", g)
}
sddl += fmt.Sprintf("(A;;GRGW;;;%s)", sid)
}
}
return sddl, nil
}
5 changes: 2 additions & 3 deletions frontend/dockerfile/dockerfile2llb/convert.go
Original file line number Diff line number Diff line change
Expand Up @@ -1340,11 +1340,10 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
copyOpts := []llb.ConstraintsOpt{
llb.Platform(*d.platform),
}
copy(copyOpts, fileOpt)
copyOpts = append(copyOpts, fileOpt...)
copyOpts = append(copyOpts, llb.ProgressGroup(pgID, pgName, true))

var mergeOpts []llb.ConstraintsOpt
copy(mergeOpts, fileOpt)
mergeOpts := append([]llb.ConstraintsOpt{}, fileOpt...)
d.cmdIndex--
mergeOpts = append(mergeOpts, llb.ProgressGroup(pgID, pgName, false), llb.WithCustomName(prefixCommand(d, "LINK "+name, d.prefixPlatform, &platform, env)))

Expand Down
Loading

0 comments on commit 2e18d70

Please sign in to comment.