Merge pull request #79 from mallardduck/update-ci

[CI] Various CI improvements

.dockerignore

@@ -1,3 +1,4 @@
 ./.dapper
 ./.cache
 ./dist
+./build

.github/workflows/e2e/scripts/create-project-namespace.sh

@@ -5,8 +5,13 @@ source $(dirname $0)/entry
 
 cd $(dirname $0)/../../../..
 
-kubectl create namespace e2e-hpo || true
-kubectl label namespace e2e-hpo field.cattle.io/projectId=p-example --overwrite
+USE_RANCHER=${USE_RANCHER:-"false"}
+if [ "$USE_RANCHER" = "true" ]; then
+ kubectl apply -f ./examples/helm-project-operator/ci/project.yaml
+fi
+
+kubectl apply -f ./examples/helm-project-operator/ci/namespace.yaml
+
 sleep "${DEFAULT_SLEEP_TIMEOUT_SECONDS}"
 if ! kubectl get namespace cattle-project-p-example; then
  echo "ERROR: Expected cattle-project-p-example namespace to exist after ${DEFAULT_SLEEP_TIMEOUT_SECONDS} seconds, not found"

.github/workflows/e2e/scripts/create-projecthelmchart.sh

@@ -5,7 +5,7 @@ source $(dirname $0)/entry
 
 cd $(dirname $0)/../../../..
 
-kubectl apply -f ./examples/helm-project-operator/ci-example.yaml
+kubectl apply -f ./examples/helm-project-operator/ci/project-helm-chart.yaml
 sleep ${DEFAULT_SLEEP_TIMEOUT_SECONDS};
 
 if ! kubectl get -n cattle-helm-system job/helm-install-project-operator-example-chart-dummy; then

.github/workflows/e2e/scripts/delete-projecthelmchart.sh

@@ -5,7 +5,7 @@ source $(dirname $0)/entry
 
 cd $(dirname $0)/../../../..
 
-kubectl delete -f ./examples/helm-project-operator/ci-example.yaml
+kubectl delete -f ./examples/helm-project-operator/ci/project-helm-chart.yaml
 if kubectl get -n cattle-helm-system job/helm-delete-project-operator-example-chart-dummy --ignore-not-found; then
  if ! kubectl wait --for=condition=complete --timeout="${KUBECTL_WAIT_TIMEOUT}" -n cattle-helm-system job/helm-delete-project-operator-example-chart-dummy; then
  echo "ERROR: Helm Uninstall Job for Example Chart never completed after ${KUBECTL_WAIT_TIMEOUT}"

.github/workflows/hl-ci.yaml

@@ -1,6 +1,7 @@
 name: "[helm-locker] CI"
 
 on:
+ workflow_call:
  pull_request:
  push:
  branches:

.github/workflows/hl-e2e.yaml

@@ -14,7 +14,7 @@ on:
  - 'pkg/helm-locker/**'
 
 env:
- CLUSTER_NAME : test-cluster
+ CLUSTER_NAME : e2e-ci-helm-locker
  K3S_VERSION : v1.27.9-k3s1
 
 jobs:
@@ -32,12 +32,12 @@ jobs:
  run : |
  curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
  sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
- - name: Set up k3d
- run : ./.github/workflows/e2e/scripts/install-k3d.sh
  - name: build
  run: BUILD_TARGET=helm-locker make build
- - name : Setup cluster
- run : CLUSTER_NAME=${{ env.CLUSTER_NAME }} K3S_VERSION=${{ env.K3S_VERSION }} ./scripts/setup-cluster.sh
+ - name : Install k3d
+ run : ./.github/workflows/e2e/scripts/install-k3d.sh
+ - name : Setup k3d cluster
+ run : ./.github/workflows/e2e/scripts/setup-cluster.sh
  # temporary hack to run the helm-locker controller in the k3d cluster
  - name : run helm-locker
  run : |

.github/workflows/hpo-ci.yaml

@@ -2,11 +2,20 @@ name: "[helm-project-operator] ci"
 
 env:
  CGO_ENABLED: 0
- YQ_VERSION: v4.25.1
+ YQ_VERSION: v4.44.3
+
+on:
+ pull_request:
+ workflow_call:
+ inputs:
+ skip_build:
+ default: 'false'
+ required: false
+ type: string
 
-on: [ pull_request ]
 permissions:
  contents: read
+
 jobs:
  build:
  runs-on: ubuntu-latest
@@ -30,6 +39,7 @@ jobs:
  - name: Perform CI
  run : make ci
  build-images:
+ if: ${{ inputs.skip_build != 'true' }}
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@v4

.github/workflows/hpo-e2e-ci.yaml

@@ -26,11 +26,12 @@ on:
  - 'Makefile'
 
 env:
+ CLUSTER_NAME: e2e-ci-helm-project-operator
  CGO_ENABLED: 0
- YQ_VERSION: v4.25.1
+ YQ_VERSION: v4.44.3
  E2E_CI: true
  REPO: rancher
- TAG: dev
+ TAG: v0.0.0-dev.1
  APISERVER_PORT: 8001
  DEFAULT_SLEEP_TIMEOUT_SECONDS: 10
  KUBECTL_WAIT_TIMEOUT: 120s
@@ -47,6 +48,8 @@ jobs:
  k3s_version:
  # k3d version list k3s | sed 's/+/-/' | sort -h
  - ${{ github.event.inputs.k3s_version || 'v1.20.15-k3s1' }}
+ env:
+ K3S_VERSION: ${{ matrix.k3s_version }}
  steps:
  - uses: actions/checkout@v4
  with:
@@ -62,21 +65,16 @@ jobs:
  run: |
  REPO=${REPO} TAG=${TAG} ./scripts/build;
  REPO=${REPO} TAG=${TAG} ./scripts/package;
- - name: Provision k3d Cluster
- uses: AbsaOSS/k3d-action@v2
- # k3d will automatically create a network named k3d-test-cluster-1 with the range 172.18.0.0/16
- with:
- cluster-name: "e2e-ci-helm-project-operator"
- args: >-
- --agents 1
- --network "nw01"
- --image docker.io/rancher/k3s:${{matrix.k3s_version}}
+ - name : Install k3d
+ run : ./.github/workflows/e2e/scripts/install-k3d.sh
+ - name : Setup k3d cluster
+ run : ./.github/workflows/e2e/scripts/setup-cluster.sh
  - name: Import Images Into k3d
  run: |
- k3d image import ${REPO}/helm-project-operator:${TAG} -c e2e-ci-helm-project-operator;
+ k3d image import ${REPO}/helm-project-operator:${TAG} -c "$CLUSTER_NAME";
  - name: Setup kubectl context
  run: |
- kubectl config use-context k3d-e2e-ci-helm-project-operator;
+ kubectl config use-context "k3d-$CLUSTER_NAME";
  - name: Set Up Tmate Debug Session
  if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.enable_tmate == 'true' }}
  uses: mxschmitt/action-tmate@v3
@@ -103,4 +101,4 @@ jobs:
 
  - name: Delete k3d cluster
  if: always()
- run: k3d cluster delete e2e-ci-helm-project-operator
+ run: k3d cluster delete "$CLUSTER_NAME";

.github/workflows/publish.yaml

@@ -8,11 +8,65 @@ on:
 env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
+ YQ_VERSION: v4.44.3
 
 jobs:
+ helm-locker-ci:
+ uses: ./.github/workflows/hl-ci.yaml
+ helm-project-operator-ci:
+ uses: ./.github/workflows/hpo-ci.yaml
+ with:
+ skip_build: 'true'
+ goreleaser:
+ name: Build go binaries and helm chart
+ runs-on : ubuntu-latest
+ needs: [
+ helm-locker-ci,
+ helm-project-operator-ci,
+ ]
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GORELEASER_CURRENT_TAG: ${{ github.ref_name }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ - run: git fetch --force --tags
+ - name : setup Go
+ uses : actions/setup-go@v5
+ with:
+ go-version: 1.22
+ - name: Install mikefarah/yq
+ run: |
+ sudo wget https:/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq;
+ - name : Install helm
+ run : |
+ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+ chmod 700 get_helm.sh
+ ./get_helm.sh
+ helm version
+ - name: Package release helm charts
+ run: make package-helm
+ - run: mkdir -p ./build/artifacts/ && mv -v ./dist/artifacts/ ./build/
+ - name: Prepare embedded helm chart
+ run: make build-chart
+ - uses: goreleaser/goreleaser-action@v6
+ with:
+ distribution: goreleaser
+ version: latest
+ args: release --clean
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GORELEASER_CURRENT_TAG: ${{ github.ref_name }}
  push:
+ needs: [
+ helm-locker-ci,
+ helm-project-operator-ci,
+ ]
  name : Build and push helm-locker & Helm-Project-Operator images
  runs-on : ubuntu-latest
+ env:
+ TAG: ${{ needs.prebuild-env.outputs.branch_static_tag }}
  permissions:
  contents: read
  packages: write
@@ -23,18 +77,12 @@ jobs:
  uses: actions/checkout@v4
  - name: Set up Docker Buildx
  uses: docker/setup-buildx-action@v3
- - name : Setup go
- uses: actions/setup-go@v5
- with:
- go-version: 1.22
  - name: Log in to the Container registry
  uses: docker/login-action@v3
  with:
  registry: ${{ env.REGISTRY }}
  username: ${{ github.actor }}
  password: ${{ secrets.GITHUB_TOKEN }}
- - name : Build, test & validate
- run : BUILD_TARGET=helm-locker make ci
  - name: Extract metadata (tags, labels) for helm-locker image
  id: meta-locker
  uses: docker/metadata-action@v5

.gitignore

@@ -1,10 +1,12 @@
 /.dapper
 /.cache
 /bin
+/build
 !/cmd/helm-project-operator/fs/.gitkeep
 /cmd/helm-project-operator/fs/*
 /dist
 *.swp
 .idea
 /helm-project-operator
 /.vscode
+get_helm.sh

.goreleaser.yaml

@@ -0,0 +1,63 @@
+# Make sure to check the documentation at https://goreleaser.com
+version: 2 # the goreleaser config version
+before:
+ hooks:
+ - go mod tidy
+builds:
+ - id: helm-locker
+ main: ./cmd/helm-locker/main.go
+ goos:
+ - linux
+ goarch:
+ - amd64
+ - arm64
+ binary: helm-locker
+ ldflags:
+ - -extldflags
+ - -static
+ - -s
+ - -X main.Version={{.Version}} -X main.GitCommit={{.Commit}}
+ flags:
+ - -trimpath
+ env:
+ - CGO_ENABLED=0
+ - id: helm-project-operator
+ main: ./cmd/helm-project-operator/main.go
+ goos:
+ - linux
+ goarch:
+ - amd64
+ - arm64
+ binary: helm-project-operator
+ ldflags:
+ - -extldflags
+ - -static
+ - -s
+ - -X main.Version={{.Version}} -X main.GitCommit={{.Commit}}
+ flags:
+ - -trimpath
+ env:
+ - CGO_ENABLED=0
+archives:
+ - id: helm-project-operator
+ builds:
+ - helm-project-operator
+ name_template: '{{ .Binary }}-{{ .Arch }}'
+ - id: helm-locker
+ builds:
+ - helm-locker
+ name_template: '{{ .Binary }}-{{ .Arch }}'
+release:
+ prerelease: auto
+ extra_files:
+ - glob : ./build/artifacts/*.tgz
+checksum:
+ name_template: 'checksums.txt'
+snapshot:
+ version_template: "{{ incpatch .Version }}-next"
+changelog:
+ sort: asc
+ filters:
+ exclude:
+ - '^docs:'
+ - '^test:'

Dockerfile.dapper

@@ -4,6 +4,8 @@ RUN git -C / clone --branch release-v3.9.0 --depth=1 https:/rancher/
 RUN make -C /helm
 
 FROM registry.suse.com/bci/golang:1.22
+# Add buildx plugin
+COPY --from=docker.io/docker/buildx-bin:0.17.1 /buildx /usr/libexec/docker/cli-plugins/docker-buildx
 
 ARG DAPPER_HOST_ARCH
 ENV ARCH $DAPPER_HOST_ARCH
@@ -18,12 +20,20 @@ ENV YQ_VERSION=v4.30.8
 RUN wget https:/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH} -O /usr/bin/yq && chmod +x /usr/bin/yq;
 
 COPY --from=helm ./helm/bin/helm /usr/local/bin/
+RUN if [ "${ARCH}" != "s390x" ]; then \
+ helm plugin install https:/quintush/helm-unittest; \
+ fi
 
-ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS
+ENV DAPPER_ENV REPO TAG HELM_IMAGE_TAG HELM_CHART_VERSION HELM_CHART_VERSION_DEV CROSS CROSS_ARCH USE_DOCKER_BUILDX TARGET_ARCH
 ENV DAPPER_SOURCE /go/src/github.com/rancher/helm-project-operator/
 ENV DAPPER_OUTPUT ./bin ./dist
-ENV DAPPER_DOCKER_SOCKET true
+ENV GOCACHE /root/.cache/go-build
 ENV GOPATH /go
+ENV DAPPER_DOCKER_SOCKET true
+ENV DAPPER_RUN_ARGS --privileged
+VOLUME /var/lib/rancher/k3s
+VOLUME /var/lib/cni
+VOLUME /var/log
 ENV HOME ${DAPPER_SOURCE}
 WORKDIR ${DAPPER_SOURCE}
 

Makefile

@@ -1,8 +1,25 @@
-TARGETS := $(shell ls scripts)
+TARGETS := $(shell ls scripts|grep -ve "^util-\|entry")
+LOCAL_TARGETS := $(addprefix local-,$(TARGETS))
 
+.dapper:
+ @echo Downloading dapper
+ @curl -sL https://releases.rancher.com/dapper/latest/dapper-$$(uname -s)-$$(uname -m) > .dapper.tmp
+ @@chmod +x .dapper.tmp
+ @./.dapper.tmp -v
+ @mv .dapper.tmp .dapper
+
+# Default behavior for targets without dapper
 $(TARGETS):
- ./scripts/$@
+ @scripts/$@
+
+# Behavior for targets prefixed with "local-" using dapper
+$(LOCAL_TARGETS): local-%: .dapper
+ ./.dapper $(@:local-%=%)
 
 .DEFAULT_GOAL := default
+.PHONY: $(TARGETS) $(LOCAL_TARGETS) list
 
-.PHONY: $(TARGETS)
+list:
+ @LC_ALL=C $(MAKE) -pRrq -f $(firstword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/(^|\n)# Files(\n|$$)/,/(^|\n)# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | grep -E -v -e '^[^[:alnum:]]' -e '^$@$$'
+# IMPORTANT: The line above must be indented by (at least one)
+# *actual TAB character* - *spaces* do *not* work.