From 3a4b78b8745fbf00cec2cb473b48e63c151b73e5 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 13:46:15 -0400 Subject: [PATCH 01/15] Adjust ci to use new webhook project label method --- .../workflows/e2e/scripts/create-project-namespace.sh | 9 +++++++-- .github/workflows/e2e/scripts/create-projecthelmchart.sh | 2 +- .github/workflows/e2e/scripts/delete-projecthelmchart.sh | 2 +- examples/helm-project-operator/ci/namespace.yaml | 8 ++++++++ .../{ci-example.yaml => ci/project-helm-chart.yaml} | 0 examples/helm-project-operator/ci/project.yaml | 8 ++++++++ 6 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 examples/helm-project-operator/ci/namespace.yaml rename examples/helm-project-operator/{ci-example.yaml => ci/project-helm-chart.yaml} (100%) create mode 100644 examples/helm-project-operator/ci/project.yaml diff --git a/.github/workflows/e2e/scripts/create-project-namespace.sh b/.github/workflows/e2e/scripts/create-project-namespace.sh index 480afcc..3a4bef3 100755 --- a/.github/workflows/e2e/scripts/create-project-namespace.sh +++ b/.github/workflows/e2e/scripts/create-project-namespace.sh @@ -5,8 +5,13 @@ source $(dirname $0)/entry cd $(dirname $0)/../../../.. -kubectl create namespace e2e-hpo || true -kubectl label namespace e2e-hpo field.cattle.io/projectId=p-example --overwrite +USE_RANCHER=${USE_RANCHER:-"false"} +if [ "$USE_RANCHER" = "true" ]; then + kubectl apply -f ./examples/helm-project-operator/ci/project.yaml +fi + +kubectl apply -f ./examples/helm-project-operator/ci/namespace.yaml + sleep "${DEFAULT_SLEEP_TIMEOUT_SECONDS}" if ! kubectl get namespace cattle-project-p-example; then echo "ERROR: Expected cattle-project-p-example namespace to exist after ${DEFAULT_SLEEP_TIMEOUT_SECONDS} seconds, not found" diff --git a/.github/workflows/e2e/scripts/create-projecthelmchart.sh b/.github/workflows/e2e/scripts/create-projecthelmchart.sh index da65e57..af1fbbe 100755 --- a/.github/workflows/e2e/scripts/create-projecthelmchart.sh +++ b/.github/workflows/e2e/scripts/create-projecthelmchart.sh @@ -5,7 +5,7 @@ source $(dirname $0)/entry cd $(dirname $0)/../../../.. -kubectl apply -f ./examples/helm-project-operator/ci-example.yaml +kubectl apply -f ./examples/helm-project-operator/ci/project-helm-chart.yaml sleep ${DEFAULT_SLEEP_TIMEOUT_SECONDS}; if ! kubectl get -n cattle-helm-system job/helm-install-project-operator-example-chart-dummy; then diff --git a/.github/workflows/e2e/scripts/delete-projecthelmchart.sh b/.github/workflows/e2e/scripts/delete-projecthelmchart.sh index 9b42a1f..bfbb631 100755 --- a/.github/workflows/e2e/scripts/delete-projecthelmchart.sh +++ b/.github/workflows/e2e/scripts/delete-projecthelmchart.sh @@ -5,7 +5,7 @@ source $(dirname $0)/entry cd $(dirname $0)/../../../.. -kubectl delete -f ./examples/helm-project-operator/ci-example.yaml +kubectl delete -f ./examples/helm-project-operator/ci/project-helm-chart.yaml if kubectl get -n cattle-helm-system job/helm-delete-project-operator-example-chart-dummy --ignore-not-found; then if ! kubectl wait --for=condition=complete --timeout="${KUBECTL_WAIT_TIMEOUT}" -n cattle-helm-system job/helm-delete-project-operator-example-chart-dummy; then echo "ERROR: Helm Uninstall Job for Example Chart never completed after ${KUBECTL_WAIT_TIMEOUT}" diff --git a/examples/helm-project-operator/ci/namespace.yaml b/examples/helm-project-operator/ci/namespace.yaml new file mode 100644 index 0000000..39b8b1f --- /dev/null +++ b/examples/helm-project-operator/ci/namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: + field.cattle.io/projectId: local:p-example + labels: + field.cattle.io/projectId: p-example + name: e2e-hpo diff --git a/examples/helm-project-operator/ci-example.yaml b/examples/helm-project-operator/ci/project-helm-chart.yaml similarity index 100% rename from examples/helm-project-operator/ci-example.yaml rename to examples/helm-project-operator/ci/project-helm-chart.yaml diff --git a/examples/helm-project-operator/ci/project.yaml b/examples/helm-project-operator/ci/project.yaml new file mode 100644 index 0000000..708aaee --- /dev/null +++ b/examples/helm-project-operator/ci/project.yaml @@ -0,0 +1,8 @@ +apiVersion: management.cattle.io/v3 +kind: Project +metadata: + name: p-example + namespace: local +spec: + clusterName: local + displayName: HelmProjectOperator Example From 87c7f3c0f85c0333331a21375bd67ab545a82a19 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 13:47:46 -0400 Subject: [PATCH 02/15] Add dapper back into makefile flow this helps make local dev envs consistent --- .dockerignore | 1 + .gitignore | 2 ++ Dockerfile.dapper | 14 ++++++++++++-- Makefile | 15 ++++++++++++++- scripts/build-chart | 1 + 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index bec10c1..0f2ec83 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ ./.dapper ./.cache ./dist +./build \ No newline at end of file diff --git a/.gitignore b/.gitignore index 57570d0..d070ac2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /.dapper /.cache /bin +/build !/cmd/helm-project-operator/fs/.gitkeep /cmd/helm-project-operator/fs/* /dist @@ -8,3 +9,4 @@ .idea /helm-project-operator /.vscode +get_helm.sh \ No newline at end of file diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 85898ff..bf0a36b 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -4,6 +4,8 @@ RUN git -C / clone --branch release-v3.9.0 --depth=1 https://github.com/rancher/ RUN make -C /helm FROM registry.suse.com/bci/golang:1.22 +# Add buildx plugin +COPY --from=docker.io/docker/buildx-bin:0.17.1 /buildx /usr/libexec/docker/cli-plugins/docker-buildx ARG DAPPER_HOST_ARCH ENV ARCH $DAPPER_HOST_ARCH @@ -18,12 +20,20 @@ ENV YQ_VERSION=v4.30.8 RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH} -O /usr/bin/yq && chmod +x /usr/bin/yq; COPY --from=helm ./helm/bin/helm /usr/local/bin/ +RUN if [ "${ARCH}" != "s390x" ]; then \ + helm plugin install https://github.com/quintush/helm-unittest; \ + fi -ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS +ENV DAPPER_ENV REPO TAG HELM_IMAGE_TAG HELM_CHART_VERSION HELM_CHART_VERSION_DEV CROSS CROSS_ARCH USE_DOCKER_BUILDX TARGET_ARCH ENV DAPPER_SOURCE /go/src/github.com/rancher/helm-project-operator/ ENV DAPPER_OUTPUT ./bin ./dist -ENV DAPPER_DOCKER_SOCKET true +ENV GOCACHE /root/.cache/go-build ENV GOPATH /go +ENV DAPPER_DOCKER_SOCKET true +ENV DAPPER_RUN_ARGS --privileged +VOLUME /var/lib/rancher/k3s +VOLUME /var/lib/cni +VOLUME /var/log ENV HOME ${DAPPER_SOURCE} WORKDIR ${DAPPER_SOURCE} diff --git a/Makefile b/Makefile index 348fabd..d2fc48b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,20 @@ TARGETS := $(shell ls scripts) +LOCAL_TARGETS := $(addprefix local-,$(TARGETS)) +.dapper: + @echo Downloading dapper + @curl -sL https://releases.rancher.com/dapper/latest/dapper-$$(uname -s)-$$(uname -m) > .dapper.tmp + @@chmod +x .dapper.tmp + @./.dapper.tmp -v + @mv .dapper.tmp .dapper + +# Default behavior for targets without dapper $(TARGETS): - ./scripts/$@ + @scripts/$@ + +# Behavior for targets prefixed with "local-" using dapper +$(LOCAL_TARGETS): local-%: .dapper + ./.dapper $(@:local-%=%) .DEFAULT_GOAL := default diff --git a/scripts/build-chart b/scripts/build-chart index 0a65c20..aa3c9be 100755 --- a/scripts/build-chart +++ b/scripts/build-chart @@ -3,6 +3,7 @@ set -e BUILD_TARGET=${BUILD_TARGET:-"helm-project-operator"} +# We don't do this for helm-locker if [[ "${BUILD_TARGET}" != "helm-project-operator" ]]; then exit fi From 0aebe351e5445fc4112c1d61fc27192612d900ad Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 14:17:24 -0400 Subject: [PATCH 03/15] Add chart building method from BRO --- .github/workflows/hpo-e2e-ci.yaml | 2 +- charts/helm-project-operator/Chart.yaml | 6 ++--- scripts/package-helm | 29 +++++++++++++++++++++++++ scripts/version | 11 +++++++++- 4 files changed, 43 insertions(+), 5 deletions(-) create mode 100755 scripts/package-helm diff --git a/.github/workflows/hpo-e2e-ci.yaml b/.github/workflows/hpo-e2e-ci.yaml index 4d17f3c..c5ccb8d 100644 --- a/.github/workflows/hpo-e2e-ci.yaml +++ b/.github/workflows/hpo-e2e-ci.yaml @@ -30,7 +30,7 @@ env: YQ_VERSION: v4.25.1 E2E_CI: true REPO: rancher - TAG: dev + TAG: v0.0.0-dev.1 APISERVER_PORT: 8001 DEFAULT_SLEEP_TIMEOUT_SECONDS: 10 KUBECTL_WAIT_TIMEOUT: 120s diff --git a/charts/helm-project-operator/Chart.yaml b/charts/helm-project-operator/Chart.yaml index 94a3b84..404ce89 100644 --- a/charts/helm-project-operator/Chart.yaml +++ b/charts/helm-project-operator/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: helm-project-operator description: Helm Project Operator -version: 0.2.1 -appVersion: 0.2.1 +version: 9.9.9-dev +appVersion: 9.9.9-dev annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/display-name: Helm Project Operator @@ -15,4 +15,4 @@ annotations: catalog.cattle.io/os: linux,windows maintainers: - email: dan.pock@suse.com - name: mallardduck + name: Dan Pock diff --git a/scripts/package-helm b/scripts/package-helm new file mode 100755 index 0000000..fed6365 --- /dev/null +++ b/scripts/package-helm @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +function edit-charts() { + sed -i \ + -e 's/^version:.*/version: '${1}'/' \ + -e 's/^appVersion:.*/appVersion: '${2}'/' \ + build/charts/helm-project-operator/Chart.yaml +} + +function package-charts() { + helm package --debug -d ./dist/artifacts ./build/charts/helm-project-operator +} + +if ! hash helm 2>/dev/null; then + echo "Helm is not installed" + exit 1 +fi + +cd $(dirname $0)/.. +source ./scripts/version + +rm -rf build/charts +mkdir -p build/charts dist/artifacts +cp -rf charts/helm-project-operator build/charts/ + +edit-charts "${HELM_CHART_VERSION}" "${HELM_IMAGE_TAG}" +cp -a ./build/charts/helm-project-operator/ ./dist/chart +package-charts diff --git a/scripts/version b/scripts/version index 0ae2e21..a37afad 100755 --- a/scripts/version +++ b/scripts/version @@ -18,7 +18,16 @@ fi TAG=${TAG:-${VERSION}} REPO=${REPO:-rancher} +HELM_IMAGE_TAG=${HELM_IMAGE_TAG:-${TAG}} +if [ "$TAG" == "$COMMIT" ]; then + HELM_CHART_VERSION="0.0.0-dev.${COMMIT}" +else + HELM_CHART_VERSION=${HELM_IMAGE_TAG/v/} +fi + if echo $TAG | grep -q dirty; then - TAG=dev + TAG="v0.0.0-dev.1" + HELM_IMAGE_TAG=$TAG + HELM_CHART_VERSION=${HELM_CHART_VERSION_DEV:-${HELM_IMAGE_TAG/v/}} fi IMAGE=${IMAGE:-${REPO}/${BUILD_TARGET}:${TAG}} \ No newline at end of file From 2190b965e104c1e0cd4e82fdfb0ec3bc7fdd771f Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 14:17:54 -0400 Subject: [PATCH 04/15] Move codegen template to common gen dir --- {scripts => gen}/boilerplate.go.txt | 0 pkg/codegen/main.go | 2 +- pkg/helm-locker/codegen/main.go | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename {scripts => gen}/boilerplate.go.txt (100%) diff --git a/scripts/boilerplate.go.txt b/gen/boilerplate.go.txt similarity index 100% rename from scripts/boilerplate.go.txt rename to gen/boilerplate.go.txt diff --git a/pkg/codegen/main.go b/pkg/codegen/main.go index b9550eb..16b0866 100644 --- a/pkg/codegen/main.go +++ b/pkg/codegen/main.go @@ -26,7 +26,7 @@ func main() { os.Unsetenv("GOPATH") controllergen.Run(args.Options{ OutputPackage: "github.com/rancher/helm-project-operator/pkg/generated", - Boilerplate: "scripts/boilerplate.go.txt", + Boilerplate: "gen/boilerplate.go.txt", Groups: map[string]args.Group{ "helm.cattle.io": { Types: []interface{}{ diff --git a/pkg/helm-locker/codegen/main.go b/pkg/helm-locker/codegen/main.go index b5b5827..cad6df0 100644 --- a/pkg/helm-locker/codegen/main.go +++ b/pkg/helm-locker/codegen/main.go @@ -26,7 +26,7 @@ func main() { os.Unsetenv("GOPATH") controllergen.Run(args.Options{ OutputPackage: "github.com/rancher/helm-project-operator/pkg/helm-locker/generated", - Boilerplate: "scripts/boilerplate.go.txt", + Boilerplate: "gen/boilerplate.go.txt", Groups: map[string]args.Group{ "helm.cattle.io": { Types: []interface{}{ From 6c49a94336851637bfc90209fed5b92de1bb2a44 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 14:19:33 -0400 Subject: [PATCH 05/15] Move setup-cluster workflow script to .github folder --- .../workflows/e2e/scripts}/setup-cluster.sh | 0 .github/workflows/hl-e2e.yaml | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) rename {scripts => .github/workflows/e2e/scripts}/setup-cluster.sh (100%) diff --git a/scripts/setup-cluster.sh b/.github/workflows/e2e/scripts/setup-cluster.sh similarity index 100% rename from scripts/setup-cluster.sh rename to .github/workflows/e2e/scripts/setup-cluster.sh diff --git a/.github/workflows/hl-e2e.yaml b/.github/workflows/hl-e2e.yaml index 6de58f6..8d6e49c 100644 --- a/.github/workflows/hl-e2e.yaml +++ b/.github/workflows/hl-e2e.yaml @@ -32,12 +32,12 @@ jobs: run : | curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - - name: Set up k3d - run : ./.github/workflows/e2e/scripts/install-k3d.sh - name: build run: BUILD_TARGET=helm-locker make build - - name : Setup cluster - run : CLUSTER_NAME=${{ env.CLUSTER_NAME }} K3S_VERSION=${{ env.K3S_VERSION }} ./scripts/setup-cluster.sh + - name : Install k3d + run : ./.github/workflows/e2e/scripts/install-k3d.sh + - name : Setup k3d cluster + run : CLUSTER_NAME=e2e-ci-helm-project-operator K3S_VERSION=${{ matrix.k3s_version }} ./.github/workflows/e2e/scripts/setup-cluster.sh # temporary hack to run the helm-locker controller in the k3d cluster - name : run helm-locker run : | From d4ebe7d1ff9c3f756a529b2093924f77d7633463 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 14:19:57 -0400 Subject: [PATCH 06/15] stop using AbsaOSS/k3d-action --- .github/workflows/hl-e2e.yaml | 4 ++-- .github/workflows/hpo-e2e-ci.yaml | 24 +++++++++++------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/hl-e2e.yaml b/.github/workflows/hl-e2e.yaml index 8d6e49c..ac96718 100644 --- a/.github/workflows/hl-e2e.yaml +++ b/.github/workflows/hl-e2e.yaml @@ -14,7 +14,7 @@ on: - 'pkg/helm-locker/**' env: - CLUSTER_NAME : test-cluster + CLUSTER_NAME : e2e-ci-helm-locker K3S_VERSION : v1.27.9-k3s1 jobs: @@ -37,7 +37,7 @@ jobs: - name : Install k3d run : ./.github/workflows/e2e/scripts/install-k3d.sh - name : Setup k3d cluster - run : CLUSTER_NAME=e2e-ci-helm-project-operator K3S_VERSION=${{ matrix.k3s_version }} ./.github/workflows/e2e/scripts/setup-cluster.sh + run : ./.github/workflows/e2e/scripts/setup-cluster.sh # temporary hack to run the helm-locker controller in the k3d cluster - name : run helm-locker run : | diff --git a/.github/workflows/hpo-e2e-ci.yaml b/.github/workflows/hpo-e2e-ci.yaml index c5ccb8d..5f2ca54 100644 --- a/.github/workflows/hpo-e2e-ci.yaml +++ b/.github/workflows/hpo-e2e-ci.yaml @@ -26,8 +26,9 @@ on: - 'Makefile' env: + CLUSTER_NAME: e2e-ci-helm-project-operator CGO_ENABLED: 0 - YQ_VERSION: v4.25.1 + YQ_VERSION: v4.44.3 E2E_CI: true REPO: rancher TAG: v0.0.0-dev.1 @@ -47,6 +48,8 @@ jobs: k3s_version: # k3d version list k3s | sed 's/+/-/' | sort -h - ${{ github.event.inputs.k3s_version || 'v1.20.15-k3s1' }} + env: + K3S_VERSION: ${{ matrix.k3s_version }} steps: - uses: actions/checkout@v4 with: @@ -62,21 +65,16 @@ jobs: run: | REPO=${REPO} TAG=${TAG} ./scripts/build; REPO=${REPO} TAG=${TAG} ./scripts/package; - - name: Provision k3d Cluster - uses: AbsaOSS/k3d-action@v2 - # k3d will automatically create a network named k3d-test-cluster-1 with the range 172.18.0.0/16 - with: - cluster-name: "e2e-ci-helm-project-operator" - args: >- - --agents 1 - --network "nw01" - --image docker.io/rancher/k3s:${{matrix.k3s_version}} + - name : Install k3d + run : ./.github/workflows/e2e/scripts/install-k3d.sh + - name : Setup k3d cluster + run : ./.github/workflows/e2e/scripts/setup-cluster.sh - name: Import Images Into k3d run: | - k3d image import ${REPO}/helm-project-operator:${TAG} -c e2e-ci-helm-project-operator; + k3d image import ${REPO}/helm-project-operator:${TAG} -c "$CLUSTER_NAME"; - name: Setup kubectl context run: | - kubectl config use-context k3d-e2e-ci-helm-project-operator; + kubectl config use-context "k3d-$CLUSTER_NAME"; - name: Set Up Tmate Debug Session if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.enable_tmate == 'true' }} uses: mxschmitt/action-tmate@v3 @@ -103,4 +101,4 @@ jobs: - name: Delete k3d cluster if: always() - run: k3d cluster delete e2e-ci-helm-project-operator + run: k3d cluster delete "$CLUSTER_NAME"; From 71f9506ad875d733dfbd4475c5c90ea3955c6e6e Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 14:20:30 -0400 Subject: [PATCH 07/15] Add simultaneous cross arch and cross build --- scripts/build | 30 ++++++++++++++++++++++++------ scripts/version | 7 +++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/scripts/build b/scripts/build index 89f0af0..ac9c72b 100755 --- a/scripts/build +++ b/scripts/build @@ -1,7 +1,6 @@ #!/bin/bash set -e - source $(dirname $0)/version BUILD_CMD_TARGET=${BUILD_CMD_TARGET:-"./cmd/${BUILD_TARGET}/main.go"} @@ -18,11 +17,30 @@ fi LINKFLAGS="-X github.com/rancher/helm-project-operator/pkg/version.Version=$VERSION" LINKFLAGS="-X github.com/rancher/helm-project-operator/pkg/version.GitCommit=$COMMIT $LINKFLAGS" BIN_DEST="bin/${BUILD_TARGET}" -CGO_ENABLED=0 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o "${BIN_DEST}" "${BUILD_CMD_TARGET}" -if [ "$CROSS" = "true" ] && [ "$ARCH" = "amd64" ]; then - BIN_DEST="${BIN_DEST}, bin/${BUILD_TARGET}-darwin, bin/${BUILD_TARGET}-windows" - GOOS=darwin go build -ldflags "$LINKFLAGS" -o "bin/${BUILD_TARGET}-darwin" "${BUILD_CMD_TARGET}" - GOOS=windows go build -ldflags "$LINKFLAGS" -o "bin/${BUILD_TARGET}-windows" "${BUILD_CMD_TARGET}" + +ARCHES=( "$ARCH" ) +# Set CROSS_ARCH to build for the other architecture +if [ "$CROSS_ARCH" == "true" ]; then + case "$ARCH" in + amd64) XARCH=arm64 ;; + arm64) XARCH=amd64 ;; + *) echo "Unsupported ARCH of $ARCH" 1>&2 ; exit 1 + esac + ARCHES+=( "$XARCH" ) fi +for A in "${ARCHES[@]}" ; do + GOARCH="$A" CGO_ENABLED=0 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o "${BIN_DEST}-$A" "${BUILD_CMD_TARGET}" + # Set CROSS to build for other OS'es + if [ "$CROSS" = "true" ]; then + for OS in darwin windows ; do + GOARCH="$A" GOOS=$OS go build -ldflags "$LINKFLAGS" -o "${BIN_DEST}-$OS-$A" "${BUILD_CMD_TARGET}" + done + fi +done + +cd bin +ln -sf "${BUILD_TARGET}-$ARCH" "${BUILD_TARGET}" +cd .. + echo "Build complete, binary at: \`"${BIN_DEST}"\`" \ No newline at end of file diff --git a/scripts/version b/scripts/version index a37afad..a2d9739 100755 --- a/scripts/version +++ b/scripts/version @@ -15,6 +15,13 @@ else VERSION="${COMMIT}${DIRTY}" fi +ARCH=$TARGET_ARCH +if [ -z "$ARCH" ]; then + ARCH=$(go env GOHOSTARCH) +fi + +SUFFIX="-${ARCH}" + TAG=${TAG:-${VERSION}} REPO=${REPO:-rancher} From 512152970774823f33b1451a32307f76610e0f1d Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 14:22:40 -0400 Subject: [PATCH 08/15] stop running package in ci script... ...this just leads to redundant work happening in the CI increasing build times. given that we already do image steps after the `make ci` script is called in GHA we don't need to do it here too. --- scripts/ci | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ci b/scripts/ci index 2df510e..4866bca 100755 --- a/scripts/ci +++ b/scripts/ci @@ -8,4 +8,3 @@ cd $(dirname $0) ./validate ./validate-ci ./validate-chart -./package From e5f082279915de8555bbe3eaa9da3f286c7797ca Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 14:25:18 -0400 Subject: [PATCH 09/15] make CI workflows callable --- .github/workflows/hl-ci.yaml | 1 + .github/workflows/hpo-ci.yaml | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/hl-ci.yaml b/.github/workflows/hl-ci.yaml index 1d6e1d4..5ef7cf3 100644 --- a/.github/workflows/hl-ci.yaml +++ b/.github/workflows/hl-ci.yaml @@ -1,6 +1,7 @@ name: "[helm-locker] CI" on: + workflow_call: pull_request: push: branches: diff --git a/.github/workflows/hpo-ci.yaml b/.github/workflows/hpo-ci.yaml index b278651..78cdc2b 100644 --- a/.github/workflows/hpo-ci.yaml +++ b/.github/workflows/hpo-ci.yaml @@ -3,8 +3,15 @@ name: "[helm-project-operator] ci" env: CGO_ENABLED: 0 YQ_VERSION: v4.25.1 +on: + pull_request: + workflow_call: + inputs: + skip_build: + default: 'false' + required: false + type: string -on: [ pull_request ] permissions: contents: read jobs: @@ -30,6 +37,7 @@ jobs: - name: Perform CI run : make ci build-images: + if: ${{ inputs.skip_build != 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From c8dc054d422dde3f8c17673eb35b251c3808150d Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 14:25:48 -0400 Subject: [PATCH 10/15] rework publish workflow --- .github/workflows/hpo-ci.yaml | 4 ++- .github/workflows/publish.yaml | 60 ++++++++++++++++++++++++++++---- .goreleaser.yaml | 63 ++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 .goreleaser.yaml diff --git a/.github/workflows/hpo-ci.yaml b/.github/workflows/hpo-ci.yaml index 78cdc2b..cfb454e 100644 --- a/.github/workflows/hpo-ci.yaml +++ b/.github/workflows/hpo-ci.yaml @@ -2,7 +2,8 @@ name: "[helm-project-operator] ci" env: CGO_ENABLED: 0 - YQ_VERSION: v4.25.1 + YQ_VERSION: v4.44.3 + on: pull_request: workflow_call: @@ -14,6 +15,7 @@ on: permissions: contents: read + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 83e3d93..995196d 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -8,11 +8,65 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} + YQ_VERSION: v4.44.3 jobs: + helm-locker-ci: + uses: ./.github/workflows/hl-ci.yaml + helm-project-operator-ci: + uses: ./.github/workflows/hpo-ci.yaml + with: + skip_build: 'true' + goreleaser: + name: Build go binaries and helm chart + runs-on : ubuntu-latest + needs: [ + helm-locker-ci, + helm-project-operator-ci, + ] + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GORELEASER_CURRENT_TAG: ${{ github.ref_name }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: git fetch --force --tags + - name : setup Go + uses : actions/setup-go@v5 + with: + go-version: 1.22 + - name: Install mikefarah/yq + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq; + - name : Install helm + run : | + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh + helm version + - name: Package release helm charts + run: make package-helm + - run: mkdir -p ./build/artifacts/ && mv -v ./dist/artifacts/ ./build/ + - name: Prepare embedded helm chart + run: make build-chart + - uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: latest + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GORELEASER_CURRENT_TAG: ${{ github.ref_name }} push: + needs: [ + helm-locker-ci, + helm-project-operator-ci, + ] name : Build and push helm-locker & Helm-Project-Operator images runs-on : ubuntu-latest + env: + TAG: ${{ needs.prebuild-env.outputs.branch_static_tag }} permissions: contents: read packages: write @@ -23,18 +77,12 @@ jobs: uses: actions/checkout@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name : Setup go - uses: actions/setup-go@v5 - with: - go-version: 1.22 - name: Log in to the Container registry uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name : Build, test & validate - run : BUILD_TARGET=helm-locker make ci - name: Extract metadata (tags, labels) for helm-locker image id: meta-locker uses: docker/metadata-action@v5 diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..02abc77 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,63 @@ +# Make sure to check the documentation at https://goreleaser.com +version: 2 # the goreleaser config version +before: + hooks: + - go mod tidy +builds: + - id: helm-locker + main: ./cmd/helm-locker/main.go + goos: + - linux + goarch: + - amd64 + - arm64 + binary: helm-locker + ldflags: + - -extldflags + - -static + - -s + - -X main.Version={{.Version}} -X main.GitCommit={{.Commit}} + flags: + - -trimpath + env: + - CGO_ENABLED=0 + - id: helm-project-operator + main: ./cmd/helm-project-operator/main.go + goos: + - linux + goarch: + - amd64 + - arm64 + binary: helm-project-operator + ldflags: + - -extldflags + - -static + - -s + - -X main.Version={{.Version}} -X main.GitCommit={{.Commit}} + flags: + - -trimpath + env: + - CGO_ENABLED=0 +archives: + - id: helm-project-operator + builds: + - helm-project-operator + name_template: '{{ .Binary }}-{{ .Arch }}' + - id: helm-locker + builds: + - helm-locker + name_template: '{{ .Binary }}-{{ .Arch }}' +release: + prerelease: auto + extra_files: + - glob : ./build/artifacts/*.tgz +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ incpatch .Version }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' From 6d8d6eddbd2c7b01baf94c13a5bbd2621aed1205 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 14:59:01 -0400 Subject: [PATCH 11/15] Add debug to version script when called directly --- scripts/version | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/version b/scripts/version index a2d9739..ab03187 100755 --- a/scripts/version +++ b/scripts/version @@ -37,4 +37,14 @@ if echo $TAG | grep -q dirty; then HELM_IMAGE_TAG=$TAG HELM_CHART_VERSION=${HELM_CHART_VERSION_DEV:-${HELM_IMAGE_TAG/v/}} fi -IMAGE=${IMAGE:-${REPO}/${BUILD_TARGET}:${TAG}} \ No newline at end of file +IMAGE=${IMAGE:-${REPO}/${BUILD_TARGET}:${TAG}} + +function print_version_debug() { + echo "BUILD_TARGET: $BUILD_TARGET"; + echo "SUFFIX: $SUFFIX"; + echo "HELM_IMAGE_TAG: $HELM_IMAGE_TAG"; + echo "HELM_CHART_VERSION: $HELM_CHART_VERSION"; + echo "REPO: $REPO; TAG: $TAG"; + echo "IMAGE: $IMAGE"; +} +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then print_version_debug "$1"; fi \ No newline at end of file From 7d9c98493ee941f74f7c1417e596eef8a5adf97c Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 15:01:29 -0400 Subject: [PATCH 12/15] Refactor chart scripts --- scripts/build-chart | 16 ++++++++++++---- scripts/package-helm | 18 ++++-------------- scripts/util-chart | 12 ++++++++++++ 3 files changed, 28 insertions(+), 18 deletions(-) create mode 100644 scripts/util-chart diff --git a/scripts/build-chart b/scripts/build-chart index aa3c9be..c223042 100755 --- a/scripts/build-chart +++ b/scripts/build-chart @@ -9,12 +9,20 @@ if [[ "${BUILD_TARGET}" != "helm-project-operator" ]]; then fi source $(dirname $0)/version +source $(dirname $0)/util-chart cd $(dirname $0)/.. CHART=${CHART:-"project-operator-example"} -VERSION=0.0.0 -helm package charts/${CHART} --destination bin/${CHART} -base64 -i bin/${CHART}/${CHART}-${VERSION}.tgz > cmd/${BUILD_TARGET}/fs/${CHART}.tgz.base64 -rm bin/${CHART}/${CHART}-${VERSION}.tgz \ No newline at end of file +# Prepare base chart for build +rm -rf build/charts +mkdir -p build/charts dist/artifacts +cp -rf "charts/${CHART}" build/charts/ + +# Update the chart placeholders +edit-charts "build/charts/${CHART}/Chart.yaml" "${HELM_CHART_VERSION}" "${HELM_IMAGE_TAG}" +package-charts "./build/charts/${CHART}" ./dist/charts + +# Prepare chart for embedding location +base64 -i "./dist/charts/${CHART}-${HELM_CHART_VERSION}.tgz" > "cmd/${BUILD_TARGET}/fs/${CHART}.tgz.base64" diff --git a/scripts/package-helm b/scripts/package-helm index fed6365..3fad9e1 100755 --- a/scripts/package-helm +++ b/scripts/package-helm @@ -1,29 +1,19 @@ #!/bin/bash set -e -function edit-charts() { - sed -i \ - -e 's/^version:.*/version: '${1}'/' \ - -e 's/^appVersion:.*/appVersion: '${2}'/' \ - build/charts/helm-project-operator/Chart.yaml -} - -function package-charts() { - helm package --debug -d ./dist/artifacts ./build/charts/helm-project-operator -} - if ! hash helm 2>/dev/null; then echo "Helm is not installed" exit 1 fi -cd $(dirname $0)/.. +cd "$(dirname $0)/.." source ./scripts/version +source ./scripts/util-chart rm -rf build/charts mkdir -p build/charts dist/artifacts cp -rf charts/helm-project-operator build/charts/ -edit-charts "${HELM_CHART_VERSION}" "${HELM_IMAGE_TAG}" +edit-charts build/charts/helm-project-operator/Chart.yaml "${HELM_CHART_VERSION}" "${HELM_IMAGE_TAG}" cp -a ./build/charts/helm-project-operator/ ./dist/chart -package-charts +package-charts \ No newline at end of file diff --git a/scripts/util-chart b/scripts/util-chart new file mode 100644 index 0000000..cb268f9 --- /dev/null +++ b/scripts/util-chart @@ -0,0 +1,12 @@ +#!/bin/bash + +function edit-charts() { + sed -i \ + -e 's/^version:.*/version: '${2}'/' \ + -e 's/^appVersion:.*/appVersion: '${3}'/' \ + "${1}" +} + +function package-charts() { + helm package --debug "${1:-"./build/charts/helm-project-operator"}" -d "${2:-"./dist/artifacts"}" +} \ No newline at end of file From e062f9e140730fcb06384025d69d45b1e70a6499 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 16:18:19 -0400 Subject: [PATCH 13/15] Add ability to list make targets --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d2fc48b..714c52c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -TARGETS := $(shell ls scripts) +TARGETS := $(shell ls scripts|grep -ve "^util-\|entry") LOCAL_TARGETS := $(addprefix local-,$(TARGETS)) .dapper: @@ -17,5 +17,9 @@ $(LOCAL_TARGETS): local-%: .dapper ./.dapper $(@:local-%=%) .DEFAULT_GOAL := default +.PHONY: $(TARGETS) $(LOCAL_TARGETS) list -.PHONY: $(TARGETS) \ No newline at end of file +list: + @LC_ALL=C $(MAKE) -pRrq -f $(firstword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/(^|\n)# Files(\n|$$)/,/(^|\n)# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | grep -E -v -e '^[^[:alnum:]]' -e '^$@$$' +# IMPORTANT: The line above must be indented by (at least one) +# *actual TAB character* - *spaces* do *not* work. From 202170f0c892713f1829dfce0dec456d9796e9d8 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 16:30:43 -0400 Subject: [PATCH 14/15] improve script verbosity --- scripts/package | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/package b/scripts/package index c89b309..01407ce 100755 --- a/scripts/package +++ b/scripts/package @@ -5,7 +5,7 @@ source $(dirname $0)/version DOCKER_TARGET=${DOCKER_TARGET:-"-${BUILD_TARGET}"} cd $(dirname $0)/.. -echo Building ${IMAGE} ... DOCKERFILE=package/Dockerfile${DOCKER_TARGET} -docker build -f ${DOCKERFILE} -t ${IMAGE} . -echo Built ${IMAGE} +echo "Building ${IMAGE} from ${DOCKERFILE}" +docker build -f "${DOCKERFILE}" -t "${IMAGE}" . +echo "Built ${IMAGE}" From 76b77c6da822564adeb7990ae364d0ad77540f8a Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Sun, 6 Oct 2024 16:41:28 -0400 Subject: [PATCH 15/15] update goreleaser config --- .goreleaser.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 02abc77..78803d0 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -54,7 +54,7 @@ release: checksum: name_template: 'checksums.txt' snapshot: - name_template: "{{ incpatch .Version }}-next" + version_template: "{{ incpatch .Version }}-next" changelog: sort: asc filters: