From 0a69f5e81262a9fbc4fdac5ca275fca2e9f0c6aa Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sun, 16 Oct 2022 07:02:50 +0200 Subject: [PATCH 01/14] hack: bake definition with vendor targets Signed-off-by: CrazyMax --- Dockerfile | 3 ++- Makefile | 12 ++++++++- docker-bake.hcl | 28 +++++++++++++++++++++ hack/dockerfiles/vendor.Dockerfile | 40 ++++++++++++++++++++++-------- hack/update-vendor | 12 ++++----- hack/validate-vendor | 29 ---------------------- 6 files changed, 75 insertions(+), 49 deletions(-) create mode 100644 docker-bake.hcl delete mode 100755 hack/validate-vendor diff --git a/Dockerfile b/Dockerfile index b5f0d3c252a9..a4915054b5a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ # syntax=docker/dockerfile-upstream:1.4 +ARG GO_VERSION=1.19 ARG RUNC_VERSION=v1.1.4 ARG CONTAINERD_VERSION=v1.6.10 # containerd v1.5 for integration tests @@ -22,7 +23,7 @@ RUN apk add --no-cache git # xx is a helper for cross-compilation FROM --platform=$BUILDPLATFORM tonistiigi/xx@sha256:1e96844fadaa2f9aea021b2b05299bc02fe4c39a92d8e735b93e8e2b15610128 AS xx -FROM --platform=$BUILDPLATFORM golang:1.19-alpine AS golatest +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS golatest # gobuild is base stage for compiling go/cgo FROM golatest AS gobuild-base diff --git a/Makefile b/Makefile index 813fcdf0e2e0..2fd5beff1174 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,13 @@ +ifneq (, $(BUILDX_BIN)) + export BUILDX_CMD = $(BUILDX_BIN) +else ifneq (, $(shell docker buildx version)) + export BUILDX_CMD = docker buildx +else ifneq (, $(shell which buildx)) + export BUILDX_CMD = $(which buildx) +else + export BUILDX_CMD = docker buildx +endif + prefix=/usr/local bindir=$(prefix)/bin @@ -23,7 +33,7 @@ lint: ./hack/lint validate-vendor: - ./hack/validate-vendor + $(BUILDX_CMD) bake validate-vendor validate-shfmt: ./hack/validate-shfmt diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 000000000000..7259e725d6e2 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,28 @@ +variable "GO_VERSION" { + default = "1.19" +} + +target "_common" { + args = { + GO_VERSION = GO_VERSION + BUILDKIT_CONTEXT_KEEP_GIT_DIR = 1 + } +} + +group "validate" { + targets = ["validate-vendor"] +} + +target "validate-vendor" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/vendor.Dockerfile" + target = "validate" + output = ["type=cacheonly"] +} + +target "vendor" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/vendor.Dockerfile" + target = "update" + output = ["."] +} diff --git a/hack/dockerfiles/vendor.Dockerfile b/hack/dockerfiles/vendor.Dockerfile index 4108aaaecf1b..1228a97d5ca1 100644 --- a/hack/dockerfiles/vendor.Dockerfile +++ b/hack/dockerfiles/vendor.Dockerfile @@ -1,19 +1,37 @@ # syntax=docker/dockerfile-upstream:master -FROM golang:1.19-alpine AS vendored -RUN apk add --no-cache git +ARG GO_VERSION=1.19 + +FROM golang:${GO_VERSION}-alpine AS base +RUN apk add --no-cache git rsync WORKDIR /src -RUN --mount=target=/src,rw \ - --mount=target=/go/pkg/mod,type=cache \ - go mod tidy && go mod vendor && \ - mkdir /out && cp -r go.mod go.sum vendor /out + +FROM base AS vendored +RUN --mount=target=/context \ + --mount=target=.,type=tmpfs \ + --mount=target=/go/pkg/mod,type=cache <&2 'ERROR: Vendor result differs. Please vendor your package with "make vendor"' + git status --porcelain -- go.mod go.sum vendor + exit 1 + fi +EOT diff --git a/hack/update-vendor b/hack/update-vendor index 18dd3237c608..f2a6a76ab933 100755 --- a/hack/update-vendor +++ b/hack/update-vendor @@ -1,16 +1,14 @@ #!/usr/bin/env bash -. $(dirname $0)/util set -eu +: "${BUILDX_CMD=docker buildx}" + output=$(mktemp -d -t buildx-output.XXXXXXXXXX) -buildxCmd build \ - --target "update" \ - --output "type=local,dest=$output" \ - --file "./hack/dockerfiles/vendor.Dockerfile" \ - . +set -x +${BUILDX_CMD} bake --set "*.output=$output" vendor rm -rf ./vendor cp -R "$output"/out/* . -rm -rf $output +rm -rf "$output" diff --git a/hack/validate-vendor b/hack/validate-vendor deleted file mode 100755 index d60a6fe91ab9..000000000000 --- a/hack/validate-vendor +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env sh -set -eu - -case ${1:-} in - '') - . $(dirname $0)/util - buildxCmd build \ - --target validate \ - --file ./hack/dockerfiles/vendor.Dockerfile \ - . - ;; - check) - status="$(git status --porcelain -- go.mod go.sum vendor 2>/dev/null)" - diffs=$(echo "$status" | grep -v '^[RAD] ' || true) - if [ "$diffs" ]; then - { - set +x - echo 'The result of "make vendor" differs' - echo - echo "$diffs" - echo - echo 'Please vendor your package with "make vendor"' - echo - } >&2 - exit 1 - fi - echo 'Congratulations! All vendoring changes are done the right way.' - ;; -esac From 740c95809c0a660c5a0b8180da75512adff1ea59 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sun, 16 Oct 2022 07:16:05 +0200 Subject: [PATCH 02/14] hack: authors Dockerfile and bake target Signed-off-by: CrazyMax --- Makefile | 44 ++++++++++++++++++++--------- docker-bake.hcl | 14 +++++++++ hack/dockerfiles/authors.Dockerfile | 32 +++++++++++++++++++++ hack/generate-authors | 16 ----------- 4 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 hack/dockerfiles/authors.Dockerfile delete mode 100755 hack/generate-authors diff --git a/Makefile b/Makefile index 2fd5beff1174..771b96bc845c 100644 --- a/Makefile +++ b/Makefile @@ -11,46 +11,64 @@ endif prefix=/usr/local bindir=$(prefix)/bin -binaries: FORCE +.PHONY: binaries +binaries: hack/binaries -images: FORCE +.PHONY: images +images: # moby/buildkit:local and moby/buildkit:local-rootless are created on Docker hack/images local moby/buildkit TARGET=rootless hack/images local moby/buildkit -install: FORCE +.PHONY: install +install: mkdir -p $(DESTDIR)$(bindir) install bin/* $(DESTDIR)$(bindir) -clean: FORCE +.PHONY: clean +clean: rm -rf ./bin +.PHONY: test test: - ./hack/test integration gateway dockerfile + hack/test integration gateway dockerfile +.PHONY: lint lint: - ./hack/lint + hack/lint +.PHONY: validate-vendor validate-vendor: $(BUILDX_CMD) bake validate-vendor +.PHONY: validate-shfmt validate-shfmt: - ./hack/validate-shfmt + hack/validate-shfmt +.PHONY: shfmt shfmt: - ./hack/shfmt + hack/shfmt +.PHONY: validate-authors +validate-authors: + $(BUILDX_CMD) bake validate-authors + +.PHONY: validate-generated-files validate-generated-files: - ./hack/validate-generated-files + hack/validate-generated-files +.PHONY: validate-all validate-all: test lint validate-vendor validate-generated-files +.PHONY: vendor vendor: - ./hack/update-vendor + hack/update-vendor + +.PHONY: authors +authors: + $(BUILDX_CMD) bake authors +.PHONY: generated-files generated-files: ./hack/update-generated-files - -.PHONY: vendor generated-files test binaries images install clean lint validate-all validate-vendor validate-generated-files -FORCE: diff --git a/docker-bake.hcl b/docker-bake.hcl index 7259e725d6e2..776cfcac1461 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -20,9 +20,23 @@ target "validate-vendor" { output = ["type=cacheonly"] } +target "validate-authors" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/authors.Dockerfile" + target = "validate" + output = ["type=cacheonly"] +} + target "vendor" { inherits = ["_common"] dockerfile = "./hack/dockerfiles/vendor.Dockerfile" target = "update" output = ["."] } + +target "authors" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/authors.Dockerfile" + target = "update" + output = ["."] +} diff --git a/hack/dockerfiles/authors.Dockerfile b/hack/dockerfiles/authors.Dockerfile new file mode 100644 index 000000000000..9dec80168b62 --- /dev/null +++ b/hack/dockerfiles/authors.Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile-upstream:master + +FROM alpine:3.16 AS gen +RUN apk add --no-cache git +WORKDIR /src +RUN --mount=type=bind,target=. < /out/AUTHORS + cat /out/AUTHORS +EOT + +FROM scratch AS update +COPY --from=gen /out / + +FROM gen AS validate +RUN --mount=type=bind,target=.,rw <&2 'ERROR: Authors result differs. Please update with "make authors"' + git status --porcelain -- AUTHORS + exit 1 + fi +EOT diff --git a/hack/generate-authors b/hack/generate-authors deleted file mode 100755 index 8acd3637c486..000000000000 --- a/hack/generate-authors +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -eu -o pipefail -x - -cd "$(dirname "$(readlink -f "$BASH_SOURCE")")/.." - -# see also ".mailmap" for how email addresses and names are deduplicated - -{ - cat <<-'EOH' - # This file lists all individuals having contributed content to the repository. - # For how it is generated, see `scripts/generate-authors.sh`. - EOH - echo - git log --format='%aN <%aE>' | LC_ALL=C.UTF-8 sort -uf -} >AUTHORS From f82cf1a9a696799945da18d6696770310c6391c1 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sun, 16 Oct 2022 07:53:38 +0200 Subject: [PATCH 03/14] hack: lint bake target Signed-off-by: CrazyMax --- Makefile | 2 +- docker-bake.hcl | 8 +++++++- hack/dockerfiles/lint.Dockerfile | 4 +++- hack/lint | 6 ------ 4 files changed, 11 insertions(+), 9 deletions(-) delete mode 100755 hack/lint diff --git a/Makefile b/Makefile index 771b96bc845c..f2f6d55689db 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ test: .PHONY: lint lint: - hack/lint + $(BUILDX_CMD) bake lint .PHONY: validate-vendor validate-vendor: diff --git a/docker-bake.hcl b/docker-bake.hcl index 776cfcac1461..788b855ad81d 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -10,7 +10,13 @@ target "_common" { } group "validate" { - targets = ["validate-vendor"] + targets = ["lint", "validate-vendor"] +} + +target "lint" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/lint.Dockerfile" + output = ["type=cacheonly"] } target "validate-vendor" { diff --git a/hack/dockerfiles/lint.Dockerfile b/hack/dockerfiles/lint.Dockerfile index 257a9e878196..f6566fb15ab0 100644 --- a/hack/dockerfiles/lint.Dockerfile +++ b/hack/dockerfiles/lint.Dockerfile @@ -1,6 +1,8 @@ # syntax=docker/dockerfile-upstream:master -FROM golang:1.19-alpine +ARG GO_VERSION=1.19 + +FROM golang:${GO_VERSION}-alpine ENV GOFLAGS="-buildvcs=false" RUN apk add --no-cache gcc musl-dev yamllint RUN wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.48.0 diff --git a/hack/lint b/hack/lint deleted file mode 100755 index 647e85b68269..000000000000 --- a/hack/lint +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -e - -buildxCmd build --file ./hack/dockerfiles/lint.Dockerfile . From 0ac105761106bb4c138c0005f38b0362657ec583 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sun, 16 Oct 2022 08:48:12 +0200 Subject: [PATCH 04/14] ci(validate): use make Signed-off-by: CrazyMax --- .github/workflows/validate.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 0042f1c9ac2a..8ec147e9474f 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -9,13 +9,7 @@ on: push: branches: - 'master' - tags: - - 'v*' - - 'dockerfile/*' pull_request: - branches: - - 'master' - - 'v*' env: REPO_SLUG_ORIGIN: "moby/buildkit:latest" @@ -27,11 +21,11 @@ jobs: strategy: fail-fast: false matrix: - include: - - script: ./hack/lint - - script: ./hack/validate-vendor - - script: ./hack/validate-generated-files - - script: ./hack/validate-shfmt + target: + - lint + - validate-vendor + - validate-generated-files + - validate-shfmt steps: - name: Checkout @@ -46,4 +40,4 @@ jobs: - name: Run run: | - ${{ matrix.script }} + make ${{ matrix.target }} From cafdef48938f64816808683bd08dc59e725b15dd Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 21 Oct 2022 23:02:08 +0200 Subject: [PATCH 05/14] hack: binaries and cross bake targets Signed-off-by: CrazyMax --- .github/workflows/build.yml | 27 ++++++++++++----------- .gitignore | 1 - Dockerfile | 5 ++--- Makefile | 8 +++++-- docker-bake.hcl | 43 +++++++++++++++++++++++++++++++++++++ hack/binaries | 22 ------------------- hack/cross | 21 ------------------ 7 files changed, 65 insertions(+), 62 deletions(-) delete mode 100755 hack/binaries delete mode 100755 hack/cross diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57512e8e48fa..c4583c3456b2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -305,9 +305,8 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 + with: + fetch-depth: 0 - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -319,14 +318,14 @@ jobs: driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} buildkitd-flags: --debug - - name: Cross - run: | - ./hack/cross - env: - PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64 - RUNC_PLATFORMS: ${{ env.PLATFORMS }} - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} - CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} + name: Build + uses: docker/bake-action@v2 + with: + targets: binaries-cross + set: | + *.cache-from=type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} + *.cache-to=type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} + *.output=type=cacheonly release-base: runs-on: ubuntu-20.04 @@ -396,7 +395,9 @@ jobs: binaries: runs-on: ubuntu-20.04 - needs: [release-base, test, cross] + needs: + - release-base + - test steps: - name: Checkout @@ -420,7 +421,7 @@ jobs: ./hack/release-tar "${{ needs.release-base.outputs.tag }}" release-out env: PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64 - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} + CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} - name: Move artifacts run: | diff --git a/.gitignore b/.gitignore index 75c0a9be9885..bd9babd17443 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,5 @@ # please consider a global .gitignore https://help.github.com/articles/ignoring-files bin coverage -release-out .certs .tmp diff --git a/Dockerfile b/Dockerfile index a4915054b5a9..b4d66e19d3d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile-upstream:1.4 +# syntax=docker/dockerfile-upstream:master ARG GO_VERSION=1.19 ARG RUNC_VERSION=v1.1.4 @@ -91,11 +91,10 @@ RUN --mount=target=. --mount=target=/root/.cache,type=cache \ CGO_ENABLED=0 xx-go build -ldflags "$(cat /tmp/.ldflags) -extldflags '-static'" -tags "osusergo netgo static_build seccomp ${BUILDKITD_TAGS}" -o /usr/bin/buildkitd ./cmd/buildkitd && \ xx-verify --static /usr/bin/buildkitd -FROM scratch AS binaries-linux-helper +FROM scratch AS binaries-linux COPY --link --from=runc /usr/bin/runc /buildkit-runc # built from https://github.com/tonistiigi/binfmt/releases/tag/buildkit%2Fv7.0.0-29 COPY --link --from=tonistiigi/binfmt:buildkit-v7.0.0-29@sha256:5168f6c2b692b04a7c0102751220e293e109b3dc312eb22ee1a5547b42b58de6 / / -FROM binaries-linux-helper AS binaries-linux COPY --link --from=buildctl /usr/bin/buildctl / COPY --link --from=buildkitd /usr/bin/buildkitd / diff --git a/Makefile b/Makefile index f2f6d55689db..934b2621c163 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,11 @@ bindir=$(prefix)/bin .PHONY: binaries binaries: - hack/binaries + $(BUILDX_CMD) bake binaries + +.PHONY: cross +cross: + $(BUILDX_CMD) bake binaries-cross .PHONY: images images: @@ -24,7 +28,7 @@ images: .PHONY: install install: mkdir -p $(DESTDIR)$(bindir) - install bin/* $(DESTDIR)$(bindir) + install bin/build/* $(DESTDIR)$(bindir) .PHONY: clean clean: diff --git a/docker-bake.hcl b/docker-bake.hcl index 788b855ad81d..31bea7f74327 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -2,6 +2,19 @@ variable "GO_VERSION" { default = "1.19" } +variable "BUILDKITD_TAGS" { + default = "" +} + +# Defines the output folder +variable "DESTDIR" { + default = "" +} +function "bindir" { + params = [defaultdir] + result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}" +} + target "_common" { args = { GO_VERSION = GO_VERSION @@ -9,6 +22,10 @@ target "_common" { } } +group "default" { + targets = ["binaries"] +} + group "validate" { targets = ["lint", "validate-vendor"] } @@ -46,3 +63,29 @@ target "authors" { target = "update" output = ["."] } + +target "binaries" { + inherits = ["_common"] + target = "binaries" + args = { + BUILDKITD_TAGS = BUILDKITD_TAGS + } + output = [bindir("build")] +} + +target "binaries-cross" { + inherits = ["binaries"] + output = [bindir("cross")] + platforms = [ + "darwin/amd64", + "darwin/arm64", + "linux/amd64", + "linux/arm/v7", + "linux/arm64", + "linux/s390x", + "linux/ppc64le", + "linux/riscv64", + "windows/amd64", + "windows/arm64" + ] +} diff --git a/hack/binaries b/hack/binaries deleted file mode 100755 index cb4dacb2d679..000000000000 --- a/hack/binaries +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -eu - -: ${PLATFORMS=} -: ${BUILDKITD_TAGS=} - -buildkitdTagsFlags="" -if [ -n "$BUILDKITD_TAGS" ]; then - buildkitdTagsFlags="--build-arg=BUILDKITD_TAGS=\"$BUILDKITD_TAGS\"" -fi - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -buildxCmd build $platformFlag $buildkitdTagsFlags \ - --target "binaries" \ - --output "type=local,dest=./bin/" \ - . diff --git a/hack/cross b/hack/cross deleted file mode 100755 index 1502fd266d0e..000000000000 --- a/hack/cross +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -e - -: ${PLATFORMS=} - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -if [ -n "$RUNC_PLATFORMS" ]; then - buildxCmd build $cacheFromFlags $cacheToFlags \ - --target "binaries-linux-helper" \ - --platform "$RUNC_PLATFORMS" \ - $currentcontext -fi - -buildxCmd build $platformFlag $cacheFromFlags \ - $currentcontext From bf3017006487b477e65e5394b46ebc1b426bac91 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sun, 16 Oct 2022 11:21:30 +0200 Subject: [PATCH 06/14] hack: release bake target and split binaries workflow Signed-off-by: CrazyMax --- .github/workflows/build.yml | 116 +++++++++++++++++++++++------------- Makefile | 6 ++ docker-bake.hcl | 6 ++ hack/release-tar | 28 --------- 4 files changed, 88 insertions(+), 68 deletions(-) delete mode 100755 hack/release-tar diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c4583c3456b2..4d0a9d56c8cf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,7 @@ env: TESTFLAGS: "-v --parallel=6 --timeout=30m" BUILDX_VERSION: "v0.9.1" # leave empty to use the one available on GitHub virtual environment GO_VERSION: "1.19" + DESTDIR: "./bin" jobs: base: @@ -327,14 +328,18 @@ jobs: *.cache-to=type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} *.output=type=cacheonly - release-base: + image: runs-on: ubuntu-20.04 - outputs: - tag: ${{ steps.prep.outputs.tag }} - push: ${{ steps.prep.outputs.push }} + needs: [test, cross] + strategy: + fail-fast: false + matrix: + target-stage: + - '' + - rootless steps: - - name: Prepare - id: prep + - + name: Prepare run: | TAG=pr PUSH=false @@ -348,19 +353,8 @@ jobs: TAG=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') PUSH=push fi - echo "tag=${TAG}" >>${GITHUB_OUTPUT} - echo "push=${PUSH}" >>${GITHUB_OUTPUT} - - image: - runs-on: ubuntu-20.04 - needs: [release-base, test, cross] - strategy: - fail-fast: false - matrix: - target-stage: - - '' - - rootless - steps: + echo "TAG=${TAG}" >> $GITHUB_ENV + echo "PUSH=${PUSH}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v3 @@ -379,32 +373,57 @@ jobs: buildkitd-flags: --debug - name: Login to DockerHub - if: needs.release-base.outputs.push == 'push' + if: env.PUSH == 'push' uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build ${{ needs.release-base.outputs.tag }} + name: Build ${{ env.TAG }} run: | - ./hack/images "${{ needs.release-base.outputs.tag }}" "$REPO_SLUG_TARGET" "${{ needs.release-base.outputs.push }}" + ./hack/images "${{ env.TAG }}" "$REPO_SLUG_TARGET" "${{ env.PUSH }}" env: TARGET: ${{ matrix.target-stage }} CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} type=gha,scope=image${{ matrix.target-stage }} CACHE_TO: type=gha,scope=image${{ matrix.target-stage }} + prepare-binaries: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.platforms.outputs.matrix }} + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Create matrix + id: platforms + run: | + echo "matrix=$(docker buildx bake binaries-cross --print | jq -cr '.target."binaries-cross".platforms')" >> $GITHUB_OUTPUT + - + name: Show matrix + run: | + echo ${{ steps.platforms.outputs.matrix }} + binaries: runs-on: ubuntu-20.04 needs: - - release-base - - test + - prepare-binaries + strategy: + fail-fast: false + matrix: + platform: ${{ fromJson(needs.prepare-binaries.outputs.matrix) }} steps: + - + name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v3 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 + with: + fetch-depth: 0 - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -416,23 +435,41 @@ jobs: driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} buildkitd-flags: --debug - - name: Build ${{ needs.release-base.outputs.tag }} - run: | - ./hack/release-tar "${{ needs.release-base.outputs.tag }}" release-out - env: - PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64 - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} - - - name: Move artifacts - run: | - mv ./release-out/**/* ./release-out/ + name: Build + uses: docker/bake-action@v2 + with: + targets: release + set: | + *.platform=${{ matrix.platform }} + *.cache-from=type=gha,scope=binaries-${{ env.PLATFORM_PAIR }} + *.cache-to=type=gha,scope=binaries-${{ env.PLATFORM_PAIR }} - name: Upload artifacts uses: actions/upload-artifact@v3 with: name: buildkit - path: ./release-out/* + path: ${{ env.DESTDIR }}/* if-no-files-found: error + + release: + runs-on: ubuntu-latest + needs: + - binaries + - test + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Download artifacts + uses: actions/download-artifact@v3 + with: + name: buildkit + path: ${{ env.DESTDIR }} + - + name: List artifacts + run: | + tree -nh ${{ env.DESTDIR }} - name: GitHub Release if: startsWith(github.ref, 'refs/tags/v') @@ -441,8 +478,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: draft: true - files: ./release-out/* - name: ${{ needs.release-base.outputs.tag }} + files: ${{ env.DESTDIR }}/* frontend-base: runs-on: ubuntu-20.04 diff --git a/Makefile b/Makefile index 934b2621c163..9d098c6e6f6e 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,12 @@ binaries: cross: $(BUILDX_CMD) bake binaries-cross +.PHONY: release +release: + $(BUILDX_CMD) bake release + mv -f $(CURDIR)/bin/release/**/* $(CURDIR)/bin/release/ + find $(CURDIR)/bin/release -type d -empty -delete + .PHONY: images images: # moby/buildkit:local and moby/buildkit:local-rootless are created on Docker diff --git a/docker-bake.hcl b/docker-bake.hcl index 31bea7f74327..3488f08c1efe 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -89,3 +89,9 @@ target "binaries-cross" { "windows/arm64" ] } + +target "release" { + inherits = ["binaries-cross"] + target = "release" + output = [bindir("release")] +} diff --git a/hack/release-tar b/hack/release-tar deleted file mode 100755 index bfad2d5336e7..000000000000 --- a/hack/release-tar +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -TAG=$1 -OUT=$2 - -. $(dirname $0)/util -set -eu -o pipefail - -: ${PLATFORMS=} - -usage() { - echo "usage: ./hack/release-tar " - exit 1 -} - -if [ -z "$TAG" ] || [ -z "$OUT" ]; then - usage -fi - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -buildxCmd build $platformFlag $cacheFromFlags \ - --target release \ - --output "type=local,dest=$OUT" \ - $currentcontext From 3ab41d1cb5701341f282194129c7904496073b76 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 25 Nov 2022 14:41:28 +0100 Subject: [PATCH 07/14] hack: generated-files bake target Signed-off-by: CrazyMax --- Makefile | 4 ++-- docker-bake.hcl | 16 +++++++++++++++- hack/update-generated-files | 15 --------------- hack/validate-generated-files | 10 ---------- 4 files changed, 17 insertions(+), 28 deletions(-) delete mode 100755 hack/update-generated-files delete mode 100755 hack/validate-generated-files diff --git a/Makefile b/Makefile index 9d098c6e6f6e..5af8af33d452 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ validate-authors: .PHONY: validate-generated-files validate-generated-files: - hack/validate-generated-files + $(BUILDX_CMD) bake validate-generated-files .PHONY: validate-all validate-all: test lint validate-vendor validate-generated-files @@ -81,4 +81,4 @@ authors: .PHONY: generated-files generated-files: - ./hack/update-generated-files + $(BUILDX_CMD) bake generated-files diff --git a/docker-bake.hcl b/docker-bake.hcl index 3488f08c1efe..ffc7a0f0fbed 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -27,7 +27,7 @@ group "default" { } group "validate" { - targets = ["lint", "validate-vendor"] + targets = ["lint", "validate-vendor", "validate-generated-files"] } target "lint" { @@ -43,6 +43,13 @@ target "validate-vendor" { output = ["type=cacheonly"] } +target "validate-generated-files" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/generated-files.Dockerfile" + target = "validate" + output = ["type=cacheonly"] +} + target "validate-authors" { inherits = ["_common"] dockerfile = "./hack/dockerfiles/authors.Dockerfile" @@ -57,6 +64,13 @@ target "vendor" { output = ["."] } +target "generated-files" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/generated-files.Dockerfile" + target = "update" + output = ["."] +} + target "authors" { inherits = ["_common"] dockerfile = "./hack/dockerfiles/authors.Dockerfile" diff --git a/hack/update-generated-files b/hack/update-generated-files deleted file mode 100755 index d038d7d6b632..000000000000 --- a/hack/update-generated-files +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -eu - -output=$(mktemp -d -t buildctl-output.XXXXXXXXXX) - -buildxCmd build \ - --target "update" \ - --output "type=local,dest=$output" \ - --file "./hack/dockerfiles/generated-files.Dockerfile" \ - . - -cp -R "$output/." . -rm -rf $output diff --git a/hack/validate-generated-files b/hack/validate-generated-files deleted file mode 100755 index 705ce5405ada..000000000000 --- a/hack/validate-generated-files +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -eu - -buildxCmd build \ - --target "validate" \ - --output "type=cacheonly" \ - --file "./hack/dockerfiles/generated-files.Dockerfile" \ - . From 41e219e4fd1f15ad9933049259f2f5de648a558a Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sun, 16 Oct 2022 11:10:31 +0200 Subject: [PATCH 08/14] hack: shfmt bake target Signed-off-by: CrazyMax --- Makefile | 18 +++++++++--------- docker-bake.hcl | 16 +++++++++++++++- hack/dockerfiles/shfmt.Dockerfile | 1 + hack/shfmt | 10 ---------- hack/validate-shfmt | 10 ---------- 5 files changed, 25 insertions(+), 30 deletions(-) delete mode 100755 hack/shfmt delete mode 100755 hack/validate-shfmt diff --git a/Makefile b/Makefile index 5af8af33d452..5079b875d150 100644 --- a/Makefile +++ b/Makefile @@ -52,14 +52,6 @@ lint: validate-vendor: $(BUILDX_CMD) bake validate-vendor -.PHONY: validate-shfmt -validate-shfmt: - hack/validate-shfmt - -.PHONY: shfmt -shfmt: - hack/shfmt - .PHONY: validate-authors validate-authors: $(BUILDX_CMD) bake validate-authors @@ -68,8 +60,12 @@ validate-authors: validate-generated-files: $(BUILDX_CMD) bake validate-generated-files +.PHONY: validate-shfmt +validate-shfmt: + $(BUILDX_CMD) bake validate-shfmt + .PHONY: validate-all -validate-all: test lint validate-vendor validate-generated-files +validate-all: test lint validate-vendor validate-generated-files validate-shfmt .PHONY: vendor vendor: @@ -82,3 +78,7 @@ authors: .PHONY: generated-files generated-files: $(BUILDX_CMD) bake generated-files + +.PHONY: shfmt +shfmt: + $(BUILDX_CMD) bake shfmt diff --git a/docker-bake.hcl b/docker-bake.hcl index ffc7a0f0fbed..df33fb838361 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -27,7 +27,7 @@ group "default" { } group "validate" { - targets = ["lint", "validate-vendor", "validate-generated-files"] + targets = ["lint", "validate-vendor", "validate-generated-files", "validate-shfmt"] } target "lint" { @@ -50,6 +50,13 @@ target "validate-generated-files" { output = ["type=cacheonly"] } +target "validate-shfmt" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/shfmt.Dockerfile" + target = "validate" + output = ["type=cacheonly"] +} + target "validate-authors" { inherits = ["_common"] dockerfile = "./hack/dockerfiles/authors.Dockerfile" @@ -71,6 +78,13 @@ target "generated-files" { output = ["."] } +target "shfmt" { + inherits = ["_common"] + dockerfile = "./hack/dockerfiles/shfmt.Dockerfile" + target = "update" + output = ["."] +} + target "authors" { inherits = ["_common"] dockerfile = "./hack/dockerfiles/authors.Dockerfile" diff --git a/hack/dockerfiles/shfmt.Dockerfile b/hack/dockerfiles/shfmt.Dockerfile index fb5db637d90f..86b3e86847a6 100644 --- a/hack/dockerfiles/shfmt.Dockerfile +++ b/hack/dockerfiles/shfmt.Dockerfile @@ -1,4 +1,5 @@ # syntax=docker/dockerfile-upstream:master + FROM mvdan/shfmt:v3.1.2-alpine AS shfmt WORKDIR /src ARG SHFMT_FLAGS="-i 2 -ci" diff --git a/hack/shfmt b/hack/shfmt deleted file mode 100755 index 0368e680e5ee..000000000000 --- a/hack/shfmt +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -. $(dirname $0)/util -set -e - -buildxCmd build \ - --target "update" \ - --output "type=local,dest=." \ - --file "./hack/dockerfiles/shfmt.Dockerfile" \ - . diff --git a/hack/validate-shfmt b/hack/validate-shfmt deleted file mode 100755 index a1e33f0f9548..000000000000 --- a/hack/validate-shfmt +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env sh - -. $(dirname $0)/util -set -e - -buildxCmd build \ - --target validate \ - --output "type=cacheonly" \ - --file ./hack/dockerfiles/shfmt.Dockerfile \ - . From 2c98286614741f1ceeca4e74394dac17e1bcf417 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sun, 16 Oct 2022 11:34:19 +0200 Subject: [PATCH 09/14] hack: remove ci first pass script Signed-off-by: CrazyMax --- .github/workflows/build.yml | 36 ++++++++++++++--------------------- .github/workflows/dockerd.yml | 3 +-- docker-bake.hcl | 6 ++++++ hack/build_ci_first_pass | 32 ------------------------------- 4 files changed, 21 insertions(+), 56 deletions(-) delete mode 100755 hack/build_ci_first_pass diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4d0a9d56c8cf..0c18dd0eb1c3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,7 +29,6 @@ env: DF_REPO_SLUG_TARGET: "docker/dockerfile-upstream" PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le,linux/riscv64" CACHE_GHA_SCOPE_IT: "integration-tests" - CACHE_GHA_SCOPE_BINARIES: "binaries" CACHE_GHA_SCOPE_CROSS: "cross" TESTFLAGS: "-v --parallel=6 --timeout=30m" BUILDX_VERSION: "v0.9.1" # leave empty to use the one available on GitHub virtual environment @@ -37,15 +36,12 @@ env: DESTDIR: "./bin" jobs: - base: + prepare-test: runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v3 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -56,24 +52,19 @@ jobs: version: ${{ env.BUILDX_VERSION }} driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} buildkitd-flags: --debug - - - name: Build ${{ env.CACHE_GHA_SCOPE_BINARIES }} - run: | - ./hack/build_ci_first_pass binaries - env: - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} - CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} - name: Build ${{ env.CACHE_GHA_SCOPE_IT }} - run: | - ./hack/build_ci_first_pass integration-tests - env: - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} - CACHE_TO: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} + uses: docker/bake-action@v2 + with: + targets: integration-tests-base + set: | + *.cache-from=type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} + *.cache-to=type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} test: runs-on: ubuntu-20.04 - needs: [base] + needs: + - prepare-test strategy: fail-fast: false matrix: @@ -127,7 +118,7 @@ jobs: TEST_COVERAGE: 1 TESTPKGS: ${{ matrix.pkg }} SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }} - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} + CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} - name: Upload coverage file uses: actions/upload-artifact@v3 @@ -137,7 +128,8 @@ jobs: test-nydus: runs-on: ubuntu-20.04 - needs: [base] + needs: + - prepare-test strategy: fail-fast: false matrix: @@ -188,7 +180,7 @@ jobs: test-s3: runs-on: ubuntu-20.04 needs: - - base + - prepare-test steps: - name: Checkout @@ -211,7 +203,7 @@ jobs: test-azblob: runs-on: ubuntu-20.04 needs: - - base + - prepare-test steps: - name: Checkout diff --git a/.github/workflows/dockerd.yml b/.github/workflows/dockerd.yml index 16b8942ca241..a7c5f5e97639 100644 --- a/.github/workflows/dockerd.yml +++ b/.github/workflows/dockerd.yml @@ -12,7 +12,6 @@ on: env: REPO_SLUG_ORIGIN: "moby/buildkit:latest" CACHE_GHA_SCOPE_IT: "integration-tests" - CACHE_GHA_SCOPE_BINARIES: "binaries" TESTFLAGS: "-v --parallel=1 --timeout=30m" BUILDX_VERSION: "v0.9.1" # leave empty to use the one available on GitHub virtual environment @@ -140,4 +139,4 @@ jobs: TESTPKGS: "${{ matrix.pkg }}" TESTFLAGS: "${{ env.TESTFLAGS }} --run=//worker=dockerd$" SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}" - CACHE_FROM: "type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }}" + CACHE_FROM: "type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }}" diff --git a/docker-bake.hcl b/docker-bake.hcl index df33fb838361..95dcdaff3573 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -123,3 +123,9 @@ target "release" { target = "release" output = [bindir("release")] } + +target "integration-tests-base" { + inherits = ["_common"] + target = "integration-tests-base" + output = ["type=cacheonly"] +} diff --git a/hack/build_ci_first_pass b/hack/build_ci_first_pass deleted file mode 100755 index 1d05c7e44812..000000000000 --- a/hack/build_ci_first_pass +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -TYP=$1 - -. $(dirname $0)/util -set -e - -usage() { - echo "usage: ./hack/build_ci_first_pass " - exit 1 -} - -if [ -z "$TYP" ]; then - usage -fi - -case $TYP in - "binaries") - buildxCmd build $cacheFromFlags $cacheToFlags \ - --target "binaries" \ - $currentcontext - ;; - "integration-tests") - buildxCmd build $cacheFromFlags $cacheToFlags \ - --target "integration-tests-base" \ - $currentcontext - ;; - *) - echo >&2 "Unknown type $TYP" - exit 1 - ;; -esac From f676426c4295e39ab0dbd4335f4a6e5257b78129 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sun, 16 Oct 2022 12:42:38 +0200 Subject: [PATCH 10/14] hack: image bake target Signed-off-by: CrazyMax --- .github/workflows/build.yml | 73 +++++++++++++++++++---------- Makefile | 4 +- docker-bake.hcl | 27 +++++++++++ hack/images | 92 ------------------------------------- hack/tags | 23 ++++++++++ 5 files changed, 101 insertions(+), 118 deletions(-) delete mode 100755 hack/images create mode 100755 hack/tags diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0c18dd0eb1c3..7ac26ca7d084 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -322,37 +322,45 @@ jobs: image: runs-on: ubuntu-20.04 - needs: [test, cross] + needs: + - cross + - test strategy: fail-fast: false matrix: - target-stage: + target: - '' - rootless steps: + - + name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Prepare run: | - TAG=pr + TAGS=pr + TAG_SUFFIX= PUSH=false + if [ -n "${{ matrix.target }}" ]; then + TAG_SUFFIX=-${{ matrix.target }} + fi if [ "${{ github.event_name }}" = "schedule" ]; then - TAG=nightly - PUSH=push + TAGS=nightly${TAG_SUFFIX} + PUSH=true elif [[ $GITHUB_REF == refs/tags/v* ]]; then - TAG=${GITHUB_REF#refs/tags/} - PUSH=push + TAGS="$(./hack/tags ${{ matrix.target }})" + PUSH=true elif [[ $GITHUB_REF == refs/heads/* ]]; then - TAG=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - PUSH=push + TAGS=${GITHUB_REF#refs/heads/}${TAG_SUFFIX} + PUSH=true fi - echo "TAG=${TAG}" >> $GITHUB_ENV + echo "TAGS<> $GITHUB_ENV + echo "$TAGS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "TAG_SUFFIX=${TAG_SUFFIX}" >> $GITHUB_ENV echo "PUSH=${PUSH}" >> $GITHUB_ENV - - - name: Checkout - uses: actions/checkout@v3 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -364,20 +372,37 @@ jobs: driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} buildkitd-flags: --debug - - name: Login to DockerHub - if: env.PUSH == 'push' + name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.REPO_SLUG_TARGET }} + tags: | + ${{ env.TAGS }} + bake-target: meta-helper + - + name: Login to Docker Hub + if: env.PUSH == 'true' uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build ${{ env.TAG }} - run: | - ./hack/images "${{ env.TAG }}" "$REPO_SLUG_TARGET" "${{ env.PUSH }}" + name: Build + uses: docker/bake-action@v2 + with: + files: | + ./docker-bake.hcl + ${{ steps.meta.outputs.bake-file }} + targets: image-all + set: | + *.cache-from=type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} + *.cache-from=type=gha,scope=image${{ env.TAG_SUFFIX }} + *.cache-to=type=gha,scope=image${{ env.TAG_SUFFIX }} + *.output=type=image,buildinfo-attrs=true,push=${{ env.PUSH }} env: - TARGET: ${{ matrix.target-stage }} - CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} type=gha,scope=image${{ matrix.target-stage }} - CACHE_TO: type=gha,scope=image${{ matrix.target-stage }} + IMAGE_TARGET: ${{ matrix.target }} prepare-binaries: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 5079b875d150..78fc462c03c7 100644 --- a/Makefile +++ b/Makefile @@ -28,8 +28,8 @@ release: .PHONY: images images: # moby/buildkit:local and moby/buildkit:local-rootless are created on Docker - hack/images local moby/buildkit - TARGET=rootless hack/images local moby/buildkit + $(BUILDX_CMD) bake image + IMAGE_TARGET=rootless $(BUILDX_CMD) bake image .PHONY: install install: diff --git a/docker-bake.hcl b/docker-bake.hcl index 95dcdaff3573..1beb54387bad 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -6,6 +6,10 @@ variable "BUILDKITD_TAGS" { default = "" } +variable "IMAGE_TARGET" { + default = "" +} + # Defines the output folder variable "DESTDIR" { default = "" @@ -15,6 +19,11 @@ function "bindir" { result = DESTDIR != "" ? DESTDIR : "./bin/${defaultdir}" } +# Special target: https://github.com/docker/metadata-action#bake-definition +target "meta-helper" { + tags = [IMAGE_TARGET != "" ? "moby/buildkit:local-${IMAGE_TARGET}" : "moby/buildkit:local"] +} + target "_common" { args = { GO_VERSION = GO_VERSION @@ -124,6 +133,24 @@ target "release" { output = [bindir("release")] } +target "image" { + inherits = ["_common", "meta-helper"] + target = IMAGE_TARGET + output = ["type=docker,buildinfo-attrs=true"] +} + +target "image-all" { + inherits = ["image"] + platforms = [ + "linux/amd64", + "linux/arm/v7", + "linux/arm64", + "linux/s390x", + "linux/ppc64le", + "linux/riscv64" + ] +} + target "integration-tests-base" { inherits = ["_common"] target = "integration-tests-base" diff --git a/hack/images b/hack/images deleted file mode 100755 index 1b172059eef9..000000000000 --- a/hack/images +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -TAG=$1 -REPO=$2 -PUSH=$3 - -. $(dirname $0)/util -set -eu -o pipefail - -: ${PLATFORMS=} -: ${TARGET=} - -versionTag=$(git describe --always --tags --match "v[0-9]*") - -if [[ ! "$versionTag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - versionTag="" -fi - -usage() { - echo "usage: $0 [push]" - exit 1 -} - -if [ -z "$TAG" ] || [ -z "$REPO" ]; then - usage -fi - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -localmode="" -if [[ "$TAG" == "local" ]]; then - localmode="1" - if [ "$PUSH" = "push" ]; then - echo >&2 "local images cannot be pushed" - exit 1 - fi -fi - -outputFlag="--output=type=image,push=false" -if [ "$PUSH" = "push" ]; then - outputFlag="--output=type=image,buildinfo-attrs=true,push=true" -fi -if [ -n "$localmode" ]; then - outputFlag="--output=type=docker,buildinfo-attrs=true" -fi - -targetFlag="" -if [ -n "$TARGET" ]; then - targetFlag="--target=$TARGET" -fi - -tagNames="$REPO:$TAG" -if [ -n "$TARGET" ]; then - tagNames="$tagNames-$TARGET" -fi - -if [[ "$versionTag" == "$TAG" ]]; then - if [ -n "$TARGET" ]; then - tagNames="$tagNames $REPO:$TARGET" - else - tagNames="$tagNames $REPO:latest" - fi -fi - -importCacheFlags="" -for tagName in $tagNames; do - importCacheFlags="$importCacheFlags--cache-from=type=registry,ref=$tagName " -done -if [ -n "$cacheFromFlags" ]; then - importCacheFlags="$importCacheFlags$cacheFromFlags" -fi -if [ -n "$localmode" ]; then - importCacheFlags="" -fi - -exportCacheFlags="" -if [ -n "$cacheToFlags" ]; then - exportCacheFlags="$cacheToFlags" -elif [ "$PUSH" = "push" ]; then - exportCacheFlags="$exportCacheFlags--cache-to=type=inline " -fi - -tagFlags="" -for tagName in $tagNames; do - tagFlags="$tagFlags--tag=$tagName " -done - -buildxCmd build $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag \ - $currentcontext diff --git a/hack/tags b/hack/tags new file mode 100755 index 000000000000..47ff60314aa2 --- /dev/null +++ b/hack/tags @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -e + +TARGET=$1 + +tag=$(git describe --always --tags --match "v[0-9]*") + +suffix= +if [ -n "$TARGET" ]; then + suffix=-${TARGET} +fi + +tags=${tag}${suffix} +if [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + if [[ -z "$TARGET" ]]; then + tags="${tags}\nlatest" + else + tags="${tags}\n${TARGET}" + fi +fi + +echo -e "$tags" From 09fcca0441cf5d166d6a9ba9aa6f994596892bfa Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sat, 22 Oct 2022 07:32:03 +0200 Subject: [PATCH 11/14] hack: frontend bake target Signed-off-by: CrazyMax --- .github/workflows/build.yml | 107 +++++++----- Makefile | 8 +- docker-bake.hcl | 36 ++++ .../cmd/dockerfile-frontend/Dockerfile | 36 ++-- .../cmd/dockerfile-frontend/hack/detect | 29 ---- .../cmd/dockerfile-frontend/hack/release | 157 ------------------ .../cmd/dockerfile-frontend/hack/tags | 51 ++++++ .../cmd/dockerfile-frontend/hack/version | 24 +++ 8 files changed, 204 insertions(+), 244 deletions(-) delete mode 100755 frontend/dockerfile/cmd/dockerfile-frontend/hack/detect delete mode 100755 frontend/dockerfile/cmd/dockerfile-frontend/hack/release create mode 100755 frontend/dockerfile/cmd/dockerfile-frontend/hack/tags create mode 100755 frontend/dockerfile/cmd/dockerfile-frontend/hack/version diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7ac26ca7d084..72d3bfd99a6d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -497,43 +497,54 @@ jobs: draft: true files: ${{ env.DESTDIR }}/* - frontend-base: + frontend-image: runs-on: ubuntu-20.04 if: github.event_name != 'schedule' - outputs: - typ: ${{ steps.prep.outputs.typ }} - tag: ${{ steps.prep.outputs.tag }} - push: ${{ steps.prep.outputs.push }} + needs: + - test + strategy: + fail-fast: false + matrix: + channel: + - mainline + - labs steps: + - + name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Prepare - id: prep run: | - TYP=master - TAG=mainline + TAGS=pr + TAG_SUFFIX= + BUILD=true PUSH=false - if [[ $GITHUB_REF == refs/tags/dockerfile/* ]]; then - TYP=tag - TAG=${GITHUB_REF#refs/tags/} - PUSH=push + if [ "${{ matrix.channel }}" != "mainline" ]; then + TAG_SUFFIX=-${{ matrix.channel }} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + TAGS=nightly${TAG_SUFFIX} + PUSH=true + elif [[ $GITHUB_REF == refs/tags/dockerfile/* ]]; then + TAGS="$(./frontend/dockerfile/cmd/dockerfile-frontend/hack/tags ${GITHUB_REF#refs/tags/})" + PUSH=true + if [[ "${{ matrix.channel }}" != "mainline" ]] && [[ $GITHUB_REF != refs/tags/dockerfile/*-${{ matrix.channel }}* ]]; then + BUILD=false + elif [[ "${{ matrix.channel }}" == "mainline" ]] && [[ $GITHUB_REF == refs/tags/dockerfile/*-${{ matrix.channel }}* ]]; then + BUILD=false + fi elif [[ $GITHUB_REF == refs/heads/* ]]; then - PUSH=push + TAGS=${GITHUB_REF#refs/heads/}${TAG_SUFFIX} + PUSH=true fi - echo "typ=${TYP}" >>${GITHUB_OUTPUT} - echo "tag=${TAG}" >>${GITHUB_OUTPUT} - echo "push=${PUSH}" >>${GITHUB_OUTPUT} - - frontend-image: - runs-on: ubuntu-20.04 - if: github.event_name != 'schedule' - needs: [frontend-base, test] - steps: - - - name: Checkout - uses: actions/checkout@v3 - - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 + echo "TAGS<> $GITHUB_ENV + echo "$TAGS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "TAG_SUFFIX=${TAG_SUFFIX}" >> $GITHUB_ENV + echo "BUILD=${BUILD}" >> $GITHUB_ENV + echo "PUSH=${PUSH}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -545,25 +556,33 @@ jobs: driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} buildkitd-flags: --debug - - name: Login to DockerHub + name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.DF_REPO_SLUG_TARGET }} + tags: | + ${{ env.TAGS }} + bake-target: frontend-meta-helper + - + name: Login to Docker Hub uses: docker/login-action@v2 - if: needs.frontend-base.outputs.push == 'push' + if: env.PUSH == 'true' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build ${{ needs.frontend-base.outputs.typ }}/${{ needs.frontend-base.outputs.tag }} - run: | - ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" "${{ needs.frontend-base.outputs.tag }}" "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" - env: - PLATFORMS: ${{ env.PLATFORMS }},linux/mips,linux/mipsle,linux/mips64,linux/mips64le - CACHE_FROM: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} - CACHE_TO: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} - - - name: Build ${{ needs.frontend-base.outputs.typ }}/labs - if: needs.frontend-base.outputs.typ == 'master' - run: | - ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" labs "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" + name: Build + uses: docker/bake-action@v2 + with: + files: | + ./docker-bake.hcl + ${{ steps.meta.outputs.bake-file }} + targets: frontend-image-all + set: | + *.cache-from=type=gha,scope=frontend${{ env.TAG_SUFFIX }} + *.cache-to=type=gha,scope=frontend${{ env.TAG_SUFFIX }} + *.output=type=image,buildinfo-attrs=true,push=${{ env.PUSH }} env: - PLATFORMS: ${{ env.PLATFORMS }},linux/mips,linux/mipsle,linux/mips64,linux/mips64le - CACHE_FROM: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} + FRONTEND_CHANNEL: ${{ matrix.channel }} diff --git a/Makefile b/Makefile index 78fc462c03c7..4b5c5eb92bf1 100644 --- a/Makefile +++ b/Makefile @@ -26,11 +26,15 @@ release: find $(CURDIR)/bin/release -type d -empty -delete .PHONY: images -images: -# moby/buildkit:local and moby/buildkit:local-rootless are created on Docker +images: # moby/buildkit:local and moby/buildkit:local-rootless are created on Docker $(BUILDX_CMD) bake image IMAGE_TARGET=rootless $(BUILDX_CMD) bake image +.PHONY: frontends +frontends: # docker/dockerfile:local and docker/dockerfile:local-labs are created on Docker + $(BUILDX_CMD) bake frontend-image + FRONTEND_CHANNEL=labs $(BUILDX_CMD) bake frontend-image + .PHONY: install install: mkdir -p $(DESTDIR)$(bindir) diff --git a/docker-bake.hcl b/docker-bake.hcl index 1beb54387bad..89d9821ce130 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -10,6 +10,13 @@ variable "IMAGE_TARGET" { default = "" } +variable "FRONTEND_CHANNEL" { + default = "mainline" +} +variable "FRONTEND_BUILDTAGS" { + default = "" +} + # Defines the output folder variable "DESTDIR" { default = "" @@ -23,6 +30,9 @@ function "bindir" { target "meta-helper" { tags = [IMAGE_TARGET != "" ? "moby/buildkit:local-${IMAGE_TARGET}" : "moby/buildkit:local"] } +target "frontend-meta-helper" { + tags = [FRONTEND_CHANNEL != "mainline" ? "docker/dockerfile:local-${FRONTEND_CHANNEL}" : "docker/dockerfile:local"] +} target "_common" { args = { @@ -151,6 +161,32 @@ target "image-all" { ] } +target "frontend-image" { + inherits = ["_common", "frontend-meta-helper"] + dockerfile = "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" + args = { + CHANNEL = FRONTEND_CHANNEL + BUILDTAGS = FRONTEND_BUILDTAGS + } + output = ["type=docker,buildinfo-attrs=true"] +} + +target "frontend-image-all" { + inherits = ["frontend-image"] + platforms = [ + "linux/amd64", + "linux/arm/v7", + "linux/arm64", + "linux/mips", + "linux/mipsle", + "linux/mips64", + "linux/mips64le", + "linux/s390x", + "linux/ppc64le", + "linux/riscv64" + ] +} + target "integration-tests-base" { inherits = ["_common"] target = "integration-tests-base" diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile index 23b1fd288fa0..dc7b7c8a0f04 100644 --- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile +++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile @@ -1,9 +1,11 @@ # syntax=docker/dockerfile-upstream:master +ARG GO_VERSION="1.19" + # xx is a helper for cross-compilation FROM --platform=$BUILDPLATFORM tonistiigi/xx:master@sha256:d4254d9739ce2de9fb88e09bdc716aa0c65f0446a2a2143399f991d71136a3d4 AS xx -FROM --platform=$BUILDPLATFORM golang:1.19-alpine AS base +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS base RUN apk add git bash COPY --from=xx / / WORKDIR /src @@ -11,21 +13,32 @@ ENV GOFLAGS=-mod=vendor FROM base AS version ARG CHANNEL -# TODO: PKG should be inferred from go modules -RUN --mount=target=. \ - PKG=github.com/moby/buildkit/frontend/dockerfile/cmd/dockerfile-frontend VERSION=$(./frontend/dockerfile/cmd/dockerfile-frontend/hack/detect "$CHANNEL") REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi) \ - && echo "-X main.Version=${VERSION} -X main.Revision=${REVISION} -X main.Package=${PKG}" | tee /tmp/.ldflags \ - && echo -n "${VERSION}" | tee /tmp/.version +ARG BUILDTAGS +RUN --mount=target=. < [push]" - exit 1 -} - -if [ $# != 4 ]; then - usage -fi - -parseTag() { - local prefix=$(echo $1 | cut -d/ -f 1) - if [[ "$prefix" != "dockerfile" ]]; then - echo "invalid tag $1" - exit 1 - fi - local suffix=$(echo $1 | awk -F- '{print $NF}') - local tagf=./frontend/dockerfile/release/$suffix/tags - if [ "$sufffix" == "$1" ] || [ ! -f $tagf ]; then - suffix="mainline" - fi - - local mainTag=$(echo $1 | cut -d/ -f 2) - - publishedNames=$REPO:$mainTag - - local versioned="" - # \d.\d.\d becomes latest - if [[ "$mainTag" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - publishedNames=${publishedNames},$REPO:latest - versioned=1 - fi - - # \d.\d.\d-channel becomes - if [[ "$mainTag" =~ ^[0-9]+\.[0-9]+\.[0-9]+-$suffix$ ]] && [ -f $tagf ]; then - publishedNames=${publishedNames},$REPO:$suffix - versioned=1 - fi - - # \d.\d.\d* -> \d.\d* -> \d* (except "0") - if [ "$versioned" == "1" ]; then - publishedNames=${publishedNames},$REPO:$(echo $mainTag | sed -E 's#^([0-9]+\.[0-9]+)\.[0-9]+#\1#') - if [ "$(echo $mainTag | sed -E 's#^([0-9]+)\.[0-9]+\.[0-9]+.*$#\1#')" != "0" ]; then - publishedNames=${publishedNames},$REPO:$(echo $mainTag | sed -E 's#^([0-9]+)\.[0-9]+\.[0-9]+#\1#') - fi - fi - - TAG=$suffix -} - -TYP=$1 -TAG=$2 -REPO=$3 -PUSH=$4 - -platformFlag="" -if [ -n "$PLATFORMS" ]; then - platformFlag="--platform=$PLATFORMS" -fi - -pushFlag="push=false" -if [ "$PUSH" = "push" ]; then - pushFlag="push=true" -fi - -case $TYP in -"master") - tagf=./frontend/dockerfile/release/$TAG/tags - if [ ! -f $tagf ]; then - echo "invalid release $TAG" - exit 1 - fi - - buildTags=$(cat $tagf) - pushTag="master" - if [ "$TAG" != "mainline" ]; then - pushTag=${pushTag}-$TAG - fi - - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags \ - --build-arg "CHANNEL=$TAG" \ - --build-arg "BUILDTAGS=$buildTags" \ - --output "type=image,name=$REPO:$pushTag,buildinfo-attrs=true,$pushFlag" \ - --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ - $currentcontext - ;; -"tag") - publishedNames="" - parseTag $TAG - tagf=./frontend/dockerfile/release/$TAG/tags - if [ ! -f $tagf ]; then - echo "no build tags found for $TAG" - exit 1 - fi - buildTags=$(cat $tagf) - - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags \ - --build-arg "CHANNEL=$TAG" \ - --build-arg "BUILDTAGS=$buildTags" \ - --output "type=image,\"name=$publishedNames\",buildinfo-attrs=true,$pushFlag" \ - --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ - $currentcontext - ;; -"daily") - # if [ -z $DAILY_TARGETS ]; then - # DAILY_TARGETS="" - # fi - - for TAG in $DAILY_TARGETS; do - - tagf=./frontend/dockerfile/release/$TAG/tags - if [ ! -f $tagf ]; then - echo "invalid release $TAG" - exit 1 - fi - buildTags=$(cat $tagf) - - # find the buildID of the last pushed image - # returns a BuildID if rebuild needed - - tmp=$(mktemp -d -t buildid.XXXXXXXXXX) - dt=$(date +%Y%m%d) - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags \ - --target "buildid" \ - --build-arg "CHANNEL=$TAG" \ - --build-arg "BUILDTAGS=$buildTags" \ - --build-arg "REPO=$REPO" \ - --build-arg "DATE=$dt" \ - --output "type=local,dest=$tmp" \ - --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ - $currentcontext - - if [ -f $tmp/buildid ]; then - buildid=$(cat $tmp/buildid) - echo "buildid: $buildid" - - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags \ - --build-arg "CHANNEL=$TAG" \ - --build-arg "BUILDTAGS=$buildTags" \ - --output "type=image,name=$REPO:$dt-$TAG,buildinfo-attrs=true,$pushFlag" \ - --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ - $currentcontext - rm $tmp/buildid - fi - rm -r $tmp - - done - - ;; -esac diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/tags b/frontend/dockerfile/cmd/dockerfile-frontend/hack/tags new file mode 100755 index 000000000000..300dd3c56c50 --- /dev/null +++ b/frontend/dockerfile/cmd/dockerfile-frontend/hack/tags @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -e + +usage() { + echo "$0 version" + exit 1 +} + +VERSION=$1 +if [ -z "$VERSION" ]; then + usage +fi + +prefix=$(echo "$VERSION" | cut -d/ -f 1) +if [ "$prefix" != "dockerfile" ]; then + echo "invalid version $VERSION" + exit 1 +fi + +suffix=$(echo "$VERSION" | awk -F- '{print $NF}') +tagf=./frontend/dockerfile/release/$suffix/tags +if [ "$suffix" = "$VERSION" ] || [ ! -f "$tagf" ]; then + suffix="mainline" +fi + +mainTag=$(echo "$VERSION" | cut -d/ -f 2) +tags=${mainTag} +versioned="" + +# \d.\d.\d becomes latest +if [[ "$mainTag" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + tags="${tags}\nlatest" + versioned=1 +fi + +# \d.\d.\d-channel becomes +if [[ "$mainTag" =~ ^[0-9]+\.[0-9]+\.[0-9]+-$suffix$ ]] && [ -f "$tagf" ]; then + tags="${tags}\n$suffix" + versioned=1 +fi + +# \d.\d.\d* -> \d.\d* -> \d* (except "0") +if [ "$versioned" == "1" ]; then + tags="${tags}\n$(echo "$mainTag" | sed -E 's#^([0-9]+\.[0-9]+)\.[0-9]+#\1#')" + if [ "$(echo "$mainTag" | sed -E 's#^([0-9]+)\.[0-9]+\.[0-9]+.*$#\1#')" != "0" ]; then + tags="${tags}\n$(echo "$mainTag" | sed -E 's#^([0-9]+)\.[0-9]+\.[0-9]+#\1#')" + fi +fi + +echo -e "$tags" diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/version b/frontend/dockerfile/cmd/dockerfile-frontend/hack/version new file mode 100755 index 000000000000..26dbcc02a8d1 --- /dev/null +++ b/frontend/dockerfile/cmd/dockerfile-frontend/hack/version @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +usage() { + echo "./version channel" + exit 1 +} + +if [ "$#" == 0 ]; then + usage +fi + +CHANNEL=$1 + +suffix= +if [ -n "$CHANNEL" ] && [ "$CHANNEL" != "mainline" ]; then + suffix=-$CHANNEL +fi + +version=$(git describe --always --tags --match "dockerfile/[0-9]*$suffix") +if [[ ! "$version" =~ "dockerfile" ]]; then + version=${version}$suffix +fi + +echo -n "$version" From acde6f0a7f7a49ed5a3ab1a7f2a291765965cabb Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 20 Oct 2022 09:03:09 +0200 Subject: [PATCH 12/14] ci: move frontend integration tests and build to a dedicated workflow Signed-off-by: CrazyMax --- .github/workflows/build.yml | 107 +-------------- .github/workflows/frontend.yml | 236 +++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 104 deletions(-) create mode 100644 .github/workflows/frontend.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72d3bfd99a6d..a2fb30330736 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,18 +6,14 @@ concurrency: on: schedule: - - cron: '0 10 * * *' # everyday at 10am + - cron: '0 10 * * *' workflow_dispatch: push: branches: - 'master' tags: - 'v*' - - 'dockerfile/*' pull_request: - branches: - - 'master' - - 'v*' paths-ignore: - 'README.md' - 'docs/**' @@ -26,9 +22,7 @@ on: env: REPO_SLUG_ORIGIN: "moby/buildkit:latest" REPO_SLUG_TARGET: "moby/buildkit" - DF_REPO_SLUG_TARGET: "docker/dockerfile-upstream" - PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le,linux/riscv64" - CACHE_GHA_SCOPE_IT: "integration-tests" + CACHE_GHA_SCOPE_IT: "build-integration-tests" CACHE_GHA_SCOPE_CROSS: "cross" TESTFLAGS: "-v --parallel=6 --timeout=30m" BUILDX_VERSION: "v0.9.1" # leave empty to use the one available on GitHub virtual environment @@ -53,7 +47,7 @@ jobs: driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} buildkitd-flags: --debug - - name: Build ${{ env.CACHE_GHA_SCOPE_IT }} + name: Build uses: docker/bake-action@v2 with: targets: integration-tests-base @@ -70,7 +64,6 @@ jobs: matrix: pkg: - ./client ./cmd/buildctl ./worker/containerd ./solver ./frontend - - ./frontend/dockerfile worker: - containerd - containerd-rootless @@ -81,10 +74,6 @@ jobs: - oci-snapshotter-stargz typ: - integration - - dockerfile - exclude: - - pkg: ./client ./cmd/buildctl ./worker/containerd ./solver ./frontend - typ: dockerfile include: - pkg: ./... skip-integration-tests: 1 @@ -496,93 +485,3 @@ jobs: with: draft: true files: ${{ env.DESTDIR }}/* - - frontend-image: - runs-on: ubuntu-20.04 - if: github.event_name != 'schedule' - needs: - - test - strategy: - fail-fast: false - matrix: - channel: - - mainline - - labs - steps: - - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Prepare - run: | - TAGS=pr - TAG_SUFFIX= - BUILD=true - PUSH=false - if [ "${{ matrix.channel }}" != "mainline" ]; then - TAG_SUFFIX=-${{ matrix.channel }} - fi - if [ "${{ github.event_name }}" = "schedule" ]; then - TAGS=nightly${TAG_SUFFIX} - PUSH=true - elif [[ $GITHUB_REF == refs/tags/dockerfile/* ]]; then - TAGS="$(./frontend/dockerfile/cmd/dockerfile-frontend/hack/tags ${GITHUB_REF#refs/tags/})" - PUSH=true - if [[ "${{ matrix.channel }}" != "mainline" ]] && [[ $GITHUB_REF != refs/tags/dockerfile/*-${{ matrix.channel }}* ]]; then - BUILD=false - elif [[ "${{ matrix.channel }}" == "mainline" ]] && [[ $GITHUB_REF == refs/tags/dockerfile/*-${{ matrix.channel }}* ]]; then - BUILD=false - fi - elif [[ $GITHUB_REF == refs/heads/* ]]; then - TAGS=${GITHUB_REF#refs/heads/}${TAG_SUFFIX} - PUSH=true - fi - echo "TAGS<> $GITHUB_ENV - echo "$TAGS" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - echo "TAG_SUFFIX=${TAG_SUFFIX}" >> $GITHUB_ENV - echo "BUILD=${BUILD}" >> $GITHUB_ENV - echo "PUSH=${PUSH}" >> $GITHUB_ENV - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - version: ${{ env.BUILDX_VERSION }} - driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} - buildkitd-flags: --debug - - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ${{ env.DF_REPO_SLUG_TARGET }} - tags: | - ${{ env.TAGS }} - bake-target: frontend-meta-helper - - - name: Login to Docker Hub - uses: docker/login-action@v2 - if: env.PUSH == 'true' - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build - uses: docker/bake-action@v2 - with: - files: | - ./docker-bake.hcl - ${{ steps.meta.outputs.bake-file }} - targets: frontend-image-all - set: | - *.cache-from=type=gha,scope=frontend${{ env.TAG_SUFFIX }} - *.cache-to=type=gha,scope=frontend${{ env.TAG_SUFFIX }} - *.output=type=image,buildinfo-attrs=true,push=${{ env.PUSH }} - env: - FRONTEND_CHANNEL: ${{ matrix.channel }} diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 000000000000..0e590302c90e --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,236 @@ +name: frontend + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + schedule: + - cron: '0 10 * * *' + workflow_dispatch: + push: + branches: + - 'master' + tags: + - 'dockerfile/*' + pull_request: + paths-ignore: + - 'README.md' + - 'docs/**' + - 'frontend/dockerfile/docs/**' + +env: + REPO_SLUG_ORIGIN: "moby/buildkit:latest" + REPO_SLUG_TARGET: "docker/dockerfile-upstream" + CACHE_GHA_SCOPE_IT: "frontend-integration-tests" + TESTFLAGS: "-v --parallel=6 --timeout=30m" + BUILDX_VERSION: "v0.9.1" # leave empty to use the one available on GitHub virtual environment + GO_VERSION: "1.19" + DESTDIR: "./bin" + +jobs: + prepare-test: + runs-on: ubuntu-20.04 + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + version: ${{ env.BUILDX_VERSION }} + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + buildkitd-flags: --debug + - + name: Build + uses: docker/bake-action@v2 + with: + targets: integration-tests-base + set: | + *.cache-from=type=gha,scope=${{ github.workflow }}-${{ env.CACHE_GHA_SCOPE_IT }} + *.cache-to=type=gha,scope=${{ github.workflow }}-${{ env.CACHE_GHA_SCOPE_IT }} + + test: + runs-on: ubuntu-20.04 + needs: + - prepare-test + strategy: + fail-fast: false + matrix: + pkg: + - ./frontend/dockerfile + worker: + - containerd + - containerd-rootless + - containerd-1.5 + - containerd-1.4 + - containerd-snapshotter-stargz + - oci + - oci-rootless + - oci-snapshotter-stargz + typ: + - integration + - dockerfile + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + version: ${{ env.BUILDX_VERSION }} + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + buildkitd-flags: --debug + - + name: Test pkg=${{ matrix.pkg }} ; typ=${{ matrix.typ }} ; worker=${{ matrix.worker }} + run: | + if [ -n "${{ matrix.worker }}" ]; then + export TESTFLAGS="${TESTFLAGS} --run=//worker=${{ matrix.worker }}$" + fi + ./hack/test ${{ matrix.typ }} + mv ./coverage/coverage.txt ./coverage/coverage-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.typ }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]').txt + env: + TEST_COVERAGE: 1 + TESTPKGS: ${{ matrix.pkg }} + CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} + - + name: Upload coverage file + uses: actions/upload-artifact@v3 + with: + name: coverage + path: ./coverage + + upload-coverage: + runs-on: ubuntu-20.04 + needs: + - test + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Download coverage files + uses: actions/download-artifact@v3 + with: + name: coverage + path: ./coverage + - + name: List coverage files + uses: actions/github-script@v6 + id: files + with: + result-encoding: string + script: | + return require('fs').readdirSync('./coverage', {withFileTypes: true}) + .filter(item => !item.isDirectory()) + .map(item => `./coverage/${item.name}`) + .join(','); + - + name: Send to Codecov + uses: codecov/codecov-action@v2 + with: + files: ${{ steps.files.outputs.result }} + + image: + runs-on: ubuntu-20.04 + needs: + - test + strategy: + fail-fast: false + matrix: + channel: + - mainline + - labs + steps: + - + name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - + name: Prepare + run: | + TAGS=pr + TAG_SUFFIX= + BUILD=true + PUSH=false + if [ "${{ matrix.channel }}" != "mainline" ]; then + TAG_SUFFIX=-${{ matrix.channel }} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + TAGS=nightly${TAG_SUFFIX} + PUSH=true + elif [[ $GITHUB_REF == refs/tags/dockerfile/* ]]; then + TAGS="$(./frontend/dockerfile/cmd/dockerfile-frontend/hack/tags ${GITHUB_REF#refs/tags/})" + PUSH=true + if [[ "${{ matrix.channel }}" != "mainline" ]] && [[ $GITHUB_REF != refs/tags/dockerfile/*-${{ matrix.channel }} ]]; then + BUILD=false + elif [[ "${{ matrix.channel }}" == "mainline" ]] && [[ $GITHUB_REF == refs/tags/dockerfile/*-${{ matrix.channel }} ]]; then + BUILD=false + fi + elif [[ $GITHUB_REF == refs/heads/* ]]; then + TAGS=${GITHUB_REF#refs/heads/}${TAG_SUFFIX} + PUSH=true + fi + echo "TAGS<> $GITHUB_ENV + echo "$TAGS" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "TAG_SUFFIX=${TAG_SUFFIX}" >> $GITHUB_ENV + echo "BUILD=${BUILD}" >> $GITHUB_ENV + echo "PUSH=${PUSH}" >> $GITHUB_ENV + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + if: env.BUILD == 'true' + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + if: env.BUILD == 'true' + with: + version: ${{ env.BUILDX_VERSION }} + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + buildkitd-flags: --debug + - + name: Docker meta + id: meta + uses: docker/metadata-action@v4 + if: env.BUILD == 'true' + with: + images: | + ${{ env.REPO_SLUG_TARGET }} + tags: | + ${{ env.TAGS }} + bake-target: frontend-meta-helper + - + name: Login to Docker Hub + uses: docker/login-action@v2 + if: env.BUILD == 'true' && env.PUSH == 'true' + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - + name: Build + uses: docker/bake-action@v2 + if: env.BUILD == 'true' + with: + files: | + ./docker-bake.hcl + ${{ steps.meta.outputs.bake-file }} + targets: frontend-image-all + set: | + *.cache-from=type=gha,scope=frontend${{ env.TAG_SUFFIX }} + *.cache-to=type=gha,scope=frontend${{ env.TAG_SUFFIX }} + *.output=type=image,buildinfo-attrs=true,push=${{ env.PUSH }} + env: + FRONTEND_CHANNEL: ${{ matrix.channel }} From 60c79d05d4ef0d8ddd4bc3bcbbb33a3ac4125992 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 21 Oct 2022 22:46:39 +0200 Subject: [PATCH 13/14] update contributing docs Signed-off-by: CrazyMax --- .github/CONTRIBUTING.md | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5bc20d744c5e..7cdc926818ae 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -52,16 +52,15 @@ Dependencies: - [runc](https://github.com/opencontainers/runc) - [containerd](https://github.com/containerd/containerd) (if you want to use containerd worker) - -The following command installs `buildkitd` and `buildctl` to `/usr/local/bin`: - ```bash -make && sudo make install -``` +# installs buildkitd and buildctl to /usr/local/bin +$ make && sudo make install -To build containerized `moby/buildkit:local` and `moby/buildkit:local-rootless` images: -```bash -make images +# build moby/buildkit:local and moby/buildkit:local-rootless images +$ make images + +# build docker/dockerfile:local and docker/dockerfile:local-labs images +$ make frontends ``` ### Run the unit- and integration-tests @@ -94,13 +93,15 @@ TESTPKGS=./client TESTFLAGS="--run //worker=containerd -v" ./hack/test integrati DOCKERFILE_RELEASES=labs TESTFLAGS="--run /TestRunGlobalNetwork/worker=oci$/ -v" ./hack/test dockerfile ``` -Set `TEST_KEEP_CACHE=1` for the test framework to keep external dependant images in a docker volume -if you are repeatedly calling `./hack/test` script. This helps to avoid rate limiting on the remote registry side. +> **Note** +> +> Set `TEST_KEEP_CACHE=1` for the test framework to keep external dependant +> images in a docker volume if you are repeatedly calling `./hack/test` script. +> This helps to avoid rate limiting on the remote registry side. Updating vendored dependencies: ```bash -# update vendor.conf make vendor ``` @@ -110,8 +111,6 @@ Validating your updates before submission: make validate-all ``` - - ### Pull requests are always welcome Not sure if that typo is worth a pull request? Found a bug and know how to fix From 44172e1b25d3f09be063c3dc277d68cdcc709d15 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sat, 22 Oct 2022 01:01:19 +0200 Subject: [PATCH 14/14] ci: update codecov action to v3 Signed-off-by: CrazyMax --- .github/workflows/frontend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 0e590302c90e..efe4a9e2540b 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -138,7 +138,7 @@ jobs: .join(','); - name: Send to Codecov - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: files: ${{ steps.files.outputs.result }}