Add support for recreating objects when immutable fields are updated

.github/workflows/e2e.yaml

@@ -31,6 +31,8 @@ jobs:
  uses: fluxcd/pkg//actions/kubebuilder@main
  - name: Setup Kubectl
  uses: fluxcd/pkg/actions/kubectl@main
+ with:
+ version: 1.20.2
  - name: Run tests
  run: make test
  env:

Makefile

@@ -107,7 +107,7 @@ ifeq (, $(shell which controller-gen))
  CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
  cd $$CONTROLLER_GEN_TMP_DIR ;\
  go mod init tmp ;\
- go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 ;\
+ go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1 ;\
  rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
  }
 CONTROLLER_GEN=$(GOBIN)/controller-gen

api/go.mod

@@ -5,8 +5,8 @@ go 1.15
 require (
  github.com/fluxcd/pkg/apis/kustomize v0.0.1
  github.com/fluxcd/pkg/apis/meta v0.7.0
- github.com/fluxcd/pkg/runtime v0.8.0
+ github.com/fluxcd/pkg/runtime v0.8.3
  k8s.io/apiextensions-apiserver v0.20.2
  k8s.io/apimachinery v0.20.2
- sigs.k8s.io/controller-runtime v0.8.0
+ sigs.k8s.io/controller-runtime v0.8.2
 )

api/go.sum

@@ -92,8 +92,8 @@ github.com/fluxcd/pkg/apis/kustomize v0.0.1 h1:TkA80R0GopRY27VJqzKyS6ifiKIAfwBd7
 github.com/fluxcd/pkg/apis/kustomize v0.0.1/go.mod h1:JAFPfnRmcrAoG1gNiA8kmEXsnOBuDyZ/F5X4DAQcVV0=
 github.com/fluxcd/pkg/apis/meta v0.7.0 h1:5e8gm4OLqjuKWdrOIY5DEEsjcwzyJFK8rCDesJ+V8IY=
 github.com/fluxcd/pkg/apis/meta v0.7.0/go.mod h1:yHuY8kyGHYz22I0jQzqMMGCcHViuzC/WPdo9Gisk8Po=
-github.com/fluxcd/pkg/runtime v0.8.0 h1:cnSBZJLcXlKgjXpFFFExu+4ZncIxmPgNIx+ErLcCLnA=
-github.com/fluxcd/pkg/runtime v0.8.0/go.mod h1:tQwEN+RESjJmtwSSv7I+6bkNM9raIXpGsCjruaIVX6A=
+github.com/fluxcd/pkg/runtime v0.8.3 h1:Zjk4fyAfBdBQ4GTokjisab7KyHHczCqKSpJi8+oVrNw=
+github.com/fluxcd/pkg/runtime v0.8.3/go.mod h1:AM/hMD0mKtRqhKPU7NGDzm+3UXPpdnX8oBlcxLt11AY=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@@ -656,14 +656,15 @@ k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ=
+k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
-sigs.k8s.io/controller-runtime v0.8.0 h1:s0dYdo7lQgJiAf+alP82PRwbz+oAqL3oSyMQ18XRDOc=
-sigs.k8s.io/controller-runtime v0.8.0/go.mod h1:v9Lbj5oX443uR7GXYY46E0EE2o7k2YxQ58GxVNeXSW4=
+sigs.k8s.io/controller-runtime v0.8.2 h1:SBWmI0b3uzMIUD/BIXWNegrCeZmPJ503pOtwxY0LPHM=
+sigs.k8s.io/controller-runtime v0.8.2/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

api/v1beta1/kustomization_types.go

@@ -17,9 +17,10 @@ limitations under the License.
 package v1beta1
 
 import (
+ "time"
+
  apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
  apimeta "k8s.io/apimachinery/pkg/api/meta"
- "time"
 
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  "k8s.io/apimachinery/pkg/types"
@@ -124,10 +125,19 @@ type KustomizationSpec struct {
  Timeout *metav1.Duration `json:"timeout,omitempty"`
 
  // Validate the Kubernetes objects before applying them on the cluster.
- // The validation strategy can be 'client' (local dry-run), 'server' (APIServer dry-run) or 'none'.
+ // The validation strategy can be 'client' (local dry-run), 'server'
+ // (APIServer dry-run) or 'none'.
+ // When 'Force' is 'true', validation will fallback to 'client' if set to
+ // 'server' because server-side validation is not supported in this scenario.
  // +kubebuilder:validation:Enum=none;client;server
  // +optional
  Validation string `json:"validation,omitempty"`
+
+ // Force instructs the controller to recreate resources
+ // when patching fails due to an immutable field change.
+ // +kubebuilder:default:=false
+ // +optional
+ Force bool `json:"force,omitempty"`
 }
 
 // Decryption defines how decryption is handled for Kubernetes manifests.

config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
  annotations:
- controller-gen.kubebuilder.io/version: v0.3.0
+ controller-gen.kubebuilder.io/version: v0.4.1
  creationTimestamp: null
  name: kustomizations.kustomize.toolkit.fluxcd.io
 spec:
@@ -88,6 +88,11 @@ spec:
  - name
  type: object
  type: array
+ force:
+ default: false
+ description: Force instructs the controller to recreate resources
+ when patching fails due to an immutable field change.
+ type: boolean
  healthChecks:
  description: A list of resources to be included in the health assessment.
  items:
@@ -346,7 +351,9 @@ spec:
  validation:
  description: Validate the Kubernetes objects before applying them
  on the cluster. The validation strategy can be 'client' (local dry-run),
- 'server' (APIServer dry-run) or 'none'.
+ 'server' (APIServer dry-run) or 'none'. When 'Force' is 'true',
+ validation will fallback to 'client' if set to 'server' because
+ server-side validation is not supported in this scenario.
  enum:
  - none
  - client

controllers/kustomization_controller.go

@@ -571,8 +571,15 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku
  applyCtx, cancel := context.WithTimeout(ctx, timeout)
  defer cancel()
 
- cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --dry-run=%s --cache-dir=/tmp",
- dirPath, kustomization.GetUID(), kustomization.GetTimeout().String(), kustomization.Spec.Validation)
+ validation := kustomization.Spec.Validation
+ if validation == "server" && kustomization.Spec.Force {
+ // Use client-side validation with force
+ validation = "client"
+ (logr.FromContext(ctx)).Info(fmt.Sprintf("Server-side validation is configured, falling-back to client-side validation since 'force' is enabled"))
+ }
+
+ cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --dry-run=%s --cache-dir=/tmp --force=%t",
+ dirPath, kustomization.GetUID(), kustomization.GetTimeout().String(), validation, kustomization.Spec.Force)
 
  if kustomization.Spec.KubeConfig != nil {
  kubeConfig, err := imp.WriteKubeConfig(ctx)
@@ -610,8 +617,8 @@ func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kusto
  defer cancel()
  fieldManager := "kustomize-controller"
 
- cmd := fmt.Sprintf("cd %s && kubectl apply --field-manager=%s -f %s.yaml --timeout=%s --cache-dir=/tmp",
- dirPath, fieldManager, kustomization.GetUID(), kustomization.Spec.Interval.Duration.String())
+ cmd := fmt.Sprintf("cd %s && kubectl apply --field-manager=%s -f %s.yaml --timeout=%s --cache-dir=/tmp --force=%t",
+ dirPath, fieldManager, kustomization.GetUID(), kustomization.Spec.Interval.Duration.String(), kustomization.Spec.Force)
 
  if kustomization.Spec.KubeConfig != nil {
  kubeConfig, err := imp.WriteKubeConfig(ctx)

controllers/kustomization_controller_test.go

@@ -24,6 +24,7 @@ import (
  . "github.com/onsi/ginkgo"
  . "github.com/onsi/ginkgo/extensions/table"
  . "github.com/onsi/gomega"
+ appsv1 "k8s.io/api/apps/v1"
  corev1 "k8s.io/api/core/v1"
  apimeta "k8s.io/apimachinery/pkg/api/meta"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -197,6 +198,7 @@ var _ = Describe("KustomizationReconciler", func() {
  Suspend: false,
  Timeout: nil,
  Validation: "client",
+ Force: false,
  PostBuild: &kustomizev1.PostBuild{
  Substitute: map[string]string{"region": "eu-central-1"},
  SubstituteFrom: []kustomizev1.SubstituteReference{
@@ -282,5 +284,148 @@ metadata:
  expectRevision: "branch/commit1",
  }),
  )
+
+ Describe("Kustomization resource replacement", func() {
+ deploymentManifest := func(namespace, selector string) string {
+ return fmt.Sprintf(`---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: test
+ namespace: %s
+spec:
+ selector:
+ matchLabels:
+ app: %[2]s
+ template:
+ metadata:
+ labels:
+ app: %[2]s
+ spec:
+ containers:
+ - name: test
+ image: podinfo
+`,
+ namespace, selector)
+ }
+
+ It("should replace immutable field resource using force", func() {
+ manifests := []testserver.File{
+ {
+ Name: "deployment.yaml",
+ Body: deploymentManifest(namespace.Name, "v1"),
+ },
+ }
+ artifact, err := httpServer.ArtifactFromFiles(manifests)
+ Expect(err).NotTo(HaveOccurred())
+
+ url := fmt.Sprintf("%s/%s", httpServer.URL(), artifact)
+
+ repositoryName := types.NamespacedName{
+ Name: fmt.Sprintf("%s", randStringRunes(5)),
+ Namespace: namespace.Name,
+ }
+ repository := &sourcev1.GitRepository{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: repositoryName.Name,
+ Namespace: repositoryName.Namespace,
+ },
+ Spec: sourcev1.GitRepositorySpec{
+ URL: "https:/test/repository",
+ Interval: metav1.Duration{Duration: reconciliationInterval},
+ },
+ Status: sourcev1.GitRepositoryStatus{
+ Conditions: []metav1.Condition{
+ {
+ Type: meta.ReadyCondition,
+ Status: metav1.ConditionTrue,
+ LastTransitionTime: metav1.Now(),
+ Reason: sourcev1.GitOperationSucceedReason,
+ },
+ },
+ URL: url,
+ Artifact: &sourcev1.Artifact{
+ Path: url,
+ URL: url,
+ Revision: "v1",
+ LastUpdateTime: metav1.Now(),
+ },
+ },
+ }
+ Expect(k8sClient.Create(context.Background(), repository)).To(Succeed())
+ Expect(k8sClient.Status().Update(context.Background(), repository)).To(Succeed())
+ defer k8sClient.Delete(context.Background(), repository)
+
+ kName := types.NamespacedName{
+ Name: fmt.Sprintf("%s", randStringRunes(5)),
+ Namespace: namespace.Name,
+ }
+ k := &kustomizev1.Kustomization{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: kName.Name,
+ Namespace: kName.Namespace,
+ },
+ Spec: kustomizev1.KustomizationSpec{
+ KubeConfig: kubeconfig,
+ Interval: metav1.Duration{Duration: reconciliationInterval},
+ Path: "./",
+ Prune: true,
+ SourceRef: kustomizev1.CrossNamespaceSourceReference{
+ Kind: sourcev1.GitRepositoryKind,
+ Name: repository.Name,
+ },
+ Suspend: false,
+ Timeout: &metav1.Duration{Duration: 60 * time.Second},
+ Validation: "client",
+ Force: true,
+ },
+ }
+ Expect(k8sClient.Create(context.Background(), k)).To(Succeed())
+ defer k8sClient.Delete(context.Background(), k)
+
+ got := &kustomizev1.Kustomization{}
+ Eventually(func() bool {
+ _ = k8sClient.Get(context.Background(), kName, got)
+ c := apimeta.FindStatusCondition(got.Status.Conditions, meta.ReadyCondition)
+ return c != nil && c.Reason == meta.ReconciliationSucceededReason
+ }, timeout, interval).Should(BeTrue())
+ Expect(got.Status.LastAppliedRevision).To(Equal("v1"))
+
+ deployment := &appsv1.Deployment{}
+ Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "test", Namespace: namespace.Name}, deployment)).To(Succeed())
+ Expect(deployment.Spec.Selector.MatchLabels["app"]).To(Equal("v1"))
+
+ manifests = []testserver.File{
+ {
+ Name: "deployment.yaml",
+ Body: deploymentManifest(namespace.Name, "v2"),
+ },
+ }
+
+ artifact, err = httpServer.ArtifactFromFiles(manifests)
+ Expect(err).NotTo(HaveOccurred())
+
+ url = fmt.Sprintf("%s/%s", httpServer.URL(), artifact)
+
+ repository.Status.URL = url
+ repository.Status.Artifact = &sourcev1.Artifact{
+ Path: url,
+ URL: url,
+ Revision: "v2",
+ LastUpdateTime: metav1.Now(),
+ }
+ Expect(k8sClient.Status().Update(context.Background(), repository)).To(Succeed())
+
+ lastAppliedRev := got.Status.LastAppliedRevision
+ Eventually(func() bool {
+ _ = k8sClient.Get(context.Background(), kName, got)
+ return apimeta.IsStatusConditionTrue(got.Status.Conditions, meta.ReadyCondition) && got.Status.LastAppliedRevision != lastAppliedRev
+ }, timeout, interval).Should(BeTrue())
+ Expect(got.Status.LastAppliedRevision).To(Equal("v2"))
+
+ Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: "test", Namespace: namespace.Name}, deployment)).To(Succeed())
+ Expect(deployment.Spec.Selector.MatchLabels["app"]).To(Equal("v2"))
+ })
+ })
  })
 })

docs/api/kustomize.md

@@ -317,7 +317,23 @@ string
 <td>
 <em>(Optional)</em>
 <p>Validate the Kubernetes objects before applying them on the cluster.
-The validation strategy can be &lsquo;client&rsquo; (local dry-run), &lsquo;server&rsquo; (APIServer dry-run) or &lsquo;none&rsquo;.</p>
+The validation strategy can be &lsquo;client&rsquo; (local dry-run), &lsquo;server&rsquo;
+(APIServer dry-run) or &lsquo;none&rsquo;.
+When &lsquo;Force&rsquo; is &lsquo;true&rsquo;, validation will fallback to &lsquo;client&rsquo; if set to
+&lsquo;server&rsquo; because server-side validation is not supported in this scenario.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>force</code><br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Force instructs the controller to recreate resources
+when patching fails due to an immutable field change.</p>
 </td>
 </tr>
 </table>
@@ -760,7 +776,23 @@ string
 <td>
 <em>(Optional)</em>
 <p>Validate the Kubernetes objects before applying them on the cluster.
-The validation strategy can be &lsquo;client&rsquo; (local dry-run), &lsquo;server&rsquo; (APIServer dry-run) or &lsquo;none&rsquo;.</p>
+The validation strategy can be &lsquo;client&rsquo; (local dry-run), &lsquo;server&rsquo;
+(APIServer dry-run) or &lsquo;none&rsquo;.
+When &lsquo;Force&rsquo; is &lsquo;true&rsquo;, validation will fallback to &lsquo;client&rsquo; if set to
+&lsquo;server&rsquo; because server-side validation is not supported in this scenario.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>force</code><br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Force instructs the controller to recreate resources
+when patching fails due to an immutable field change.</p>
 </td>
 </tr>
 </tbody>

docs/spec/v1beta1/kustomization.md

@@ -98,6 +98,12 @@ type KustomizationSpec struct {
  // +kubebuilder:validation:Enum=none;client;server
  // +optional
  Validation string `json:"validation,omitempty"`
+
+ // Force instructs the controller to recreate resources
+ // when patching fails due to an immutable field change.
+ // +kubebuilder:default:=false
+ // +optional
+ Force bool `json:"force,omitempty"`
 }
 ```
 
@@ -321,6 +327,9 @@ The interval time units are `s`, `m` and `h` e.g. `interval: 5m`, the minimum va
 
 The Kustomization execution can be suspended by setting `spec.suspend` to `true`.
 
+With `spec.force` you can tell the controller to replace the resources in-cluster if the
+patching fails due to immutable fields changes.
+
 The controller can be told to reconcile the Kustomization outside of the specified interval
 by annotating the Kustomization object with: