Rework HelmRelease reconciliation logic

.github/workflows/e2e.yaml

@@ -169,7 +169,7 @@ jobs:
  kubectl -n helm-system apply -f config/testdata/$test_name
  echo -n ">>> Waiting for expected conditions"
  count=0
- until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .TestSuccess=="False" and .Ready=="False"' )" ]; do
+ until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .TestSuccess=="False" and .Ready=="False"' )" ]; do
  echo -n '.'
  sleep 5
  count=$((count + 1))
@@ -213,7 +213,7 @@ jobs:
  fi
 
  kubectl -n helm-system delete -f config/testdata/$test_name
- - name: Run install fail with remedition test
+ - name: Run install fail with remediation test
  run: |
  test_name=install-fail-remediate
  kubectl -n helm-system apply -f config/testdata/$test_name
@@ -230,21 +230,22 @@ jobs:
  done
  echo ' done'
 
- # Ensure release does not exist (was uninstalled).
- HISTORY=$(helm -n helm-system history $test_name 2>&1; exit 0)
- if [ "$HISTORY" != 'Error: release: not found' ]; then
- echo -e "Unexpected release history: $HISTORY"
+ # Ensure release was uninstalled.
+ RELEASE_STATUS=$(helm -n helm-system history $test_name -o json | jq -r 'if length == 1 then .[0].status else empty end')
+ if [ "$RELEASE_STATUS" != "uninstalled" ]; then
+ echo -e "Unexpected release status: $RELEASE_STATUS"
  exit 1
  fi
 
  kubectl -n helm-system delete -f config/testdata/$test_name
+ helm -n helm-system delete $test_name
  - name: Run install fail with retry test
  run: |
  test_name=install-fail-retry
  kubectl -n helm-system apply -f config/testdata/$test_name
  echo -n ">>> Waiting for expected conditions"
  count=0
- until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.installFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" )' )" ]; do
+ until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.installFailures == 2 and ( .status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" and .Stalled=="True" )' )" ]; do
  echo -n '.'
  sleep 5
  count=$((count + 1))
@@ -290,7 +291,7 @@ jobs:
  kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
  echo -n ">>> Waiting for expected conditions"
  count=0
- until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False"' )" ]; do
+ until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .Ready=="False" and .Stalled=="True"' )" ]; do
  echo -n '.'
  sleep 5
  count=$((count + 1))
@@ -336,7 +337,7 @@ jobs:
  kubectl -n helm-system apply -f config/testdata/$test_name/upgrade.yaml
  echo -n ">>> Waiting for expected conditions"
  count=0
- until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="False" and .TestSuccess=="False" and .Ready=="False"' )" ]; do
+ until [ 'true' == "$( kubectl -n helm-system get helmrelease/$test_name -o json | jq '.status.conditions | map( { (.type): .status } ) | add | .Released=="True" and .TestSuccess=="False" and .Ready=="False" and .Stalled=="True"' )" ]; do
  echo -n '.'
  sleep 5
  count=$((count + 1))
@@ -558,7 +559,7 @@ jobs:
  exit 1
  fi
  kubectl -n helm-system delete -f config/testdata/post-renderer-kustomize
- - name: Boostrap CRDs Upgrade Tests
+ - name: Bootstrap CRDs Upgrade Tests
  if: ${{ startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/') }}
  run: |
  REF=${{ github.ref }}

Makefile

@@ -27,6 +27,17 @@ BUILD_PLATFORMS ?= linux/amd64
 # Architecture to use envtest with
 ENVTEST_ARCH ?= amd64
 
+# Paths to download the CRD dependency to.
+CRD_DEP_ROOT ?= $(BUILD_DIR)/config/crd/bases
+
+# Keep a record of the version of the downloaded source CRDs. It is used to
+# detect and download new CRDs when the SOURCE_VER changes.
+SOURCE_VER ?= $(shell go list -m all | grep github.com/fluxcd/source-controller/api | awk '{print $$2}')
+SOURCE_CRD_VER = $(CRD_DEP_ROOT)/.src-crd-$(SOURCE_VER)
+
+# HelmChart source CRD.
+HELMCHART_SOURCE_CRD ?= $(CRD_DEP_ROOT)/source.toolkit.fluxcd.io_helmcharts.yaml
+
 # API (doc) generation utilities
 CONTROLLER_GEN_VERSION ?= v0.12.0
 GEN_API_REF_DOCS_VERSION ?= e327d0730470cbd61b06300f81c5fcf91c23c113
@@ -35,7 +46,7 @@ all: manager
 
 # Run tests
 KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)"
-test: tidy generate fmt vet manifests api-docs install-envtest
+test: tidy generate fmt vet manifests api-docs install-envtest download-crd-deps
  KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test ./... -coverprofile cover.out
  cd api; go test ./... -coverprofile cover.out
 
@@ -81,7 +92,7 @@ manifests: controller-gen
 
 # Generate API reference documentation
 api-docs: gen-crd-api-reference-docs
- $(GEN_CRD_API_REFERENCE_DOCS) -api-dir=./api/v2beta1 -config=./hack/api-docs/config.json -template-dir=./hack/api-docs/template -out-file=./docs/api/v2beta1/helm.md
+ $(GEN_CRD_API_REFERENCE_DOCS) -api-dir=./api/v2beta2 -config=./hack/api-docs/config.json -template-dir=./hack/api-docs/template -out-file=./docs/api/v2beta2/helm.md
 
 # Run go mod tidy
 tidy:
@@ -113,6 +124,24 @@ docker-build:
 docker-push:
  docker push ${IMG}
 
+# Delete previously downloaded CRDs and record the new version of the source
+# CRDs.
+$(SOURCE_CRD_VER):
+ rm -f $(CRD_DEP_ROOT)/.src-crd*
+ mkdir -p $(CRD_DEP_ROOT)
+ $(MAKE) cleanup-crd-deps
+ touch $(SOURCE_CRD_VER)
+
+$(HELMCHART_SOURCE_CRD):
+ curl -s https://raw.githubusercontent.com/fluxcd/source-controller/${SOURCE_VER}/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml > $(HELMCHART_SOURCE_CRD)
+
+# Download the CRDs the controller depends on
+download-crd-deps: $(SOURCE_CRD_VER) $(HELMCHART_SOURCE_CRD)
+
+# Delete the downloaded CRD dependencies.
+cleanup-crd-deps:
+ rm -f $(HELMCHART_SOURCE_CRD)
+
 # Find or download controller-gen
 CONTROLLER_GEN = $(GOBIN)/controller-gen
 .PHONY: controller-gen

PROJECT

@@ -4,4 +4,8 @@ resources:
 - group: helm
  kind: HelmRelease
  version: v2beta1
+- group: helm
+ kind: HelmRelease
+ version: v2beta2
+storageVersion: v2beta2
 version: "2"

api/go.mod

@@ -8,6 +8,7 @@ require (
  k8s.io/apiextensions-apiserver v0.27.4
  k8s.io/apimachinery v0.27.4
  sigs.k8s.io/controller-runtime v0.15.1
+ sigs.k8s.io/yaml v1.3.0
 )
 
 require (

api/go.sum

@@ -103,3 +103,4 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h6
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

api/v2beta1/helmrelease_types.go

@@ -28,6 +28,8 @@ import (
 
  "github.com/fluxcd/pkg/apis/kustomize"
  "github.com/fluxcd/pkg/apis/meta"
+
+ "github.com/fluxcd/helm-controller/api/v2beta2"
 )
 
 const HelmReleaseKind = "HelmRelease"
@@ -905,6 +907,46 @@ type HelmReleaseStatus struct {
  // state. It is reset after a successful reconciliation.
  // +optional
  UpgradeFailures int64 `json:"upgradeFailures,omitempty"`
+
+ // StorageNamespace is the namespace of the Helm release storage for the
+ // current release.
+ //
+ // Note: this field is provisional to the v2beta2 API, and not actively used
+ // by v2beta1 HelmReleases.
+ // +optional
+ StorageNamespace string `json:"storageNamespace,omitempty"`
+
+ // History holds the history of Helm releases performed for this HelmRelease
+ // up to the last successfully completed release.
+ //
+ // Note: this field is provisional to the v2beta2 API, and not actively used
+ // by v2beta1 HelmReleases.
+ // +optional
+ History v2beta2.Snapshots `json:"history,omitempty"`
+
+ // LastAttemptedGeneration is the last generation the controller attempted
+ // to reconcile.
+ //
+ // Note: this field is provisional to the v2beta2 API, and not actively used
+ // by v2beta1 HelmReleases.
+ // +optional
+ LastAttemptedGeneration int64 `json:"lastAttemptedGeneration,omitempty"`
+
+ // LastAttemptedConfigDigest is the digest for the config (better known as
+ // "values") of the last reconciliation attempt.
+ //
+ // Note: this field is provisional to the v2beta2 API, and not actively used
+ // by v2beta1 HelmReleases.
+ // +optional
+ LastAttemptedConfigDigest string `json:"lastAttemptedConfigDigest,omitempty"`
+
+ // LastAttemptedReleaseAction is the last release action performed for this
+ // HelmRelease. It is used to determine the active remediation strategy.
+ //
+ // Note: this field is provisional to the v2beta2 API, and not actively used
+ // by v2beta1 HelmReleases.
+ // +optional
+ LastAttemptedReleaseAction string `json:"lastAttemptedReleaseAction,omitempty"`
 }
 
 // GetHelmChart returns the namespace and name of the HelmChart.

api/v2beta2/condition_types.go

@@ -0,0 +1,98 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v2beta2
+
+const (
+ // ReleasedCondition represents the status of the last release attempt
+ // (install/upgrade/test) against the latest desired state.
+ ReleasedCondition string = "Released"
+
+ // TestSuccessCondition represents the status of the last test attempt against
+ // the latest desired state.
+ TestSuccessCondition string = "TestSuccess"
+
+ // RemediatedCondition represents the status of the last remediation attempt
+ // (uninstall/rollback) due to a failure of the last release attempt against the
+ // latest desired state.
+ RemediatedCondition string = "Remediated"
+)
+
+const (
+ // InstallSucceededReason represents the fact that the Helm install for the
+ // HelmRelease succeeded.
+ InstallSucceededReason string = "InstallSucceeded"
+
+ // InstallFailedReason represents the fact that the Helm install for the
+ // HelmRelease failed.
+ InstallFailedReason string = "InstallFailed"
+
+ // UpgradeSucceededReason represents the fact that the Helm upgrade for the
+ // HelmRelease succeeded.
+ UpgradeSucceededReason string = "UpgradeSucceeded"
+
+ // UpgradeFailedReason represents the fact that the Helm upgrade for the
+ // HelmRelease failed.
+ UpgradeFailedReason string = "UpgradeFailed"
+
+ // TestSucceededReason represents the fact that the Helm tests for the
+ // HelmRelease succeeded.
+ TestSucceededReason string = "TestSucceeded"
+
+ // TestFailedReason represents the fact that the Helm tests for the HelmRelease
+ // failed.
+ TestFailedReason string = "TestFailed"
+
+ // RollbackSucceededReason represents the fact that the Helm rollback for the
+ // HelmRelease succeeded.
+ RollbackSucceededReason string = "RollbackSucceeded"
+
+ // RollbackFailedReason represents the fact that the Helm test for the
+ // HelmRelease failed.
+ RollbackFailedReason string = "RollbackFailed"
+
+ // UninstallSucceededReason represents the fact that the Helm uninstall for the
+ // HelmRelease succeeded.
+ UninstallSucceededReason string = "UninstallSucceeded"
+
+ // UninstallFailedReason represents the fact that the Helm uninstall for the
+ // HelmRelease failed.
+ UninstallFailedReason string = "UninstallFailed"
+
+ // ArtifactFailedReason represents the fact that the artifact download for the
+ // HelmRelease failed.
+ ArtifactFailedReason string = "ArtifactFailed"
+
+ // InitFailedReason represents the fact that the initialization of the Helm
+ // configuration failed.
+ InitFailedReason string = "InitFailed"
+
+ // GetLastReleaseFailedReason represents the fact that observing the last
+ // release failed.
+ GetLastReleaseFailedReason string = "GetLastReleaseFailed"
+
+ // DependencyNotReadyReason represents the fact that
+ // one of the dependencies is not ready.
+ DependencyNotReadyReason string = "DependencyNotReady"
+
+ // ReconciliationSucceededReason represents the fact that
+ // the reconciliation succeeded.
+ ReconciliationSucceededReason string = "ReconciliationSucceeded"
+
+ // ReconciliationFailedReason represents the fact that
+ // the reconciliation failed.
+ ReconciliationFailedReason string = "ReconciliationFailed"
+)

api/v2beta2/doc.go

@@ -0,0 +1,20 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package v2beta2 contains API Schema definitions for the helm v2beta2 API group
+// +kubebuilder:object:generate=true
+// +groupName=helm.toolkit.fluxcd.io
+package v2beta2