Apply config changes in cluster

Signed-off-by: Stefan Prodan <[email protected]>

.github/workflows/tests.yaml

@@ -145,6 +145,13 @@ jobs:
  run: |
  flux check
  flux get all
+ - name: No-op apply Terraform
+ run: |
+ export TF_CLI_CONFIG_FILE="${PWD}/.terraformrc"
+ cd examples/github-via-ssh
+ terraform apply -auto-approve -var "github_token=${GITHUB_TOKEN}" -var "github_org=fluxcd-testing" -var "github_repository=${{ steps.vars.outputs.test_repo_name }}"
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }}
  - name: Destroy Terraform
  run: |
  cd examples/github-via-ssh

docs/resources/bootstrap_git.md

@@ -40,7 +40,7 @@ The following examples are available to help you use the provider:
 - `manifests_path` (String) The install manifests are built from a GitHub release or kustomize overlay if using a local path. Defaults to `https:/fluxcd/flux2/releases`.
 - `namespace` (String) The namespace scope for install manifests. Defaults to `flux-system`. It will be created if it does not exist.
 - `network_policy` (Boolean) Deny ingress access to the toolkit controllers from other namespaces using network policies. Defaults to `true`.
-- `path` (String) Path relative to the repository root, when specified the cluster sync will be scoped to this path.
+- `path` (String) Path relative to the repository root, when specified the cluster sync will be scoped to this path (immutable).
 - `recurse_submodules` (Boolean) Configures the GitRepository source to initialize and include Git submodules in the artifact it produces.
 - `registry` (String) Container registry where the toolkit images are published. Defaults to `ghcr.io/fluxcd`.
 - `secret_name` (String) Name of the secret the sync credentials can be found in or stored to. Defaults to `flux-system`.

go.mod

@@ -154,6 +154,7 @@ require (
  github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
  github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
  github.com/oklog/run v1.1.0 // indirect
+ github.com/onsi/gomega v1.32.0 // indirect
  github.com/opencontainers/go-digest v1.0.0 // indirect
  github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
  github.com/pelletier/go-toml v1.9.4 // indirect

go.sum

@@ -2,6 +2,8 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8=
 code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=

internal/provider/provider_resource_data.go

@@ -86,20 +86,20 @@ func (prd *providerResourceData) GetGitClient(ctx context.Context) (*gogit.Clien
  if err != nil {
  return nil, fmt.Errorf("could not create temporary working directory for git repository: %w", err)
  }
- client, err := gogit.NewClient(tmpDir, authOpts, clientOpts...)
+ gitClient, err := gogit.NewClient(tmpDir, authOpts, clientOpts...)
  if err != nil {
  return nil, fmt.Errorf("could not create git client: %w", err)
  }
  // TODO: Need to conditionally clone here. If repository is empty this will fail.
- _, err = client.Clone(ctx, prd.GetRepositoryURL().String(), repository.CloneConfig{
+ _, err = gitClient.Clone(ctx, prd.GetRepositoryURL().String(), repository.CloneConfig{
  CheckoutStrategy: repository.CheckoutStrategy{
  Branch: prd.git.Branch.ValueString(),
  },
  })
  if err != nil {
  return nil, fmt.Errorf("could not clone git repository: %w", err)
  }
- return client, nil
+ return gitClient, nil
 }
 
 func (prd *providerResourceData) GetBootstrapOptions() ([]bootstrap.GitOption, error) {

internal/provider/resource_bootstrap_git.go

@@ -30,6 +30,8 @@ import (
  "strings"
  "time"
 
+ "github.com/fluxcd/flux2/v2/pkg/manifestgen"
+ "github.com/fluxcd/pkg/runtime/conditions"
  "github.com/google/go-containerregistry/pkg/name"
  "github.com/hashicorp/terraform-plugin-framework-validators/setvalidator"
  "github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator"
@@ -47,8 +49,10 @@ import (
  "github.com/hashicorp/terraform-plugin-log/tflog"
  appsv1 "k8s.io/api/apps/v1"
  networkingv1 "k8s.io/api/networking/v1"
+ apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
  k8serrors "k8s.io/apimachinery/pkg/api/errors"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ apitypes "k8s.io/apimachinery/pkg/types"
  "sigs.k8s.io/controller-runtime/pkg/client"
  "sigs.k8s.io/kustomize/api/konfig"
 
@@ -256,7 +260,7 @@ func (r *bootstrapGitResource) Schema(ctx context.Context, req resource.SchemaRe
  Default: booldefault.StaticBool(defaultOpts.NetworkPolicy),
  },
  "path": schema.StringAttribute{
- Description: "Path relative to the repository root, when specified the cluster sync will be scoped to this path.",
+ Description: "Path relative to the repository root, when specified the cluster sync will be scoped to this path (immutable).",
  Optional: true,
  },
  "recurse_submodules": schema.BoolAttribute{
@@ -356,7 +360,6 @@ func (r bootstrapGitResource) ModifyPlan(ctx context.Context, req resource.Modif
  resp.Diagnostics.Append(diags...)
 }
 
-// TODO: If kustomization file exists and not all resource files exist bootstrap will fail. This is because kustomize build is run.
 func (r *bootstrapGitResource) Create(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) {
  if r.prd == nil {
  resp.Diagnostics.AddError(missingConfiguration, bootstrapGitResourceMissingConfigError)
@@ -411,7 +414,7 @@ func (r *bootstrapGitResource) Create(ctx context.Context, req resource.CreateRe
  resp.Diagnostics.AddError("Could not get bootstrap options", err.Error())
  return
  }
- b, err := bootstrap.NewPlainGitProvider(gitClient, kubeClient, bootstrapOpts...)
+ bootstrapProvider, err := bootstrap.NewPlainGitProvider(gitClient, kubeClient, bootstrapOpts...)
  if err != nil {
  resp.Diagnostics.AddError("Could not create bootstrap provider", err.Error())
  return
@@ -439,7 +442,7 @@ func (r *bootstrapGitResource) Create(ctx context.Context, req resource.CreateRe
  }
 
  manifestsBase := ""
- err = bootstrap.Run(ctx, b, manifestsBase, installOpts, secretOpts, syncOpts, 2*time.Second, timeout)
+ err = bootstrap.Run(ctx, bootstrapProvider, manifestsBase, installOpts, secretOpts, syncOpts, 2*time.Second, timeout)
  if err != nil {
  resp.Diagnostics.AddError("Bootstrap run error", err.Error())
  return
@@ -449,13 +452,13 @@ func (r *bootstrapGitResource) Create(ctx context.Context, req resource.CreateRe
  repositoryFiles := map[string]string{}
  files := []string{install.MakeDefaultOptions().ManifestFile, sync.MakeDefaultOptions().ManifestFile, konfig.DefaultKustomizationFileName()}
  for _, f := range files {
- path := filepath.Join(data.Path.ValueString(), data.Namespace.ValueString(), f)
- b, err := os.ReadFile(filepath.Join(gitClient.Path(), path))
+ filePath := filepath.Join(data.Path.ValueString(), data.Namespace.ValueString(), f)
+ b, err := os.ReadFile(filepath.Join(gitClient.Path(), filePath))
  if err != nil {
  resp.Diagnostics.AddError("Could not read repository state", err.Error())
  return
  }
- repositoryFiles[path] = string(b)
+ repositoryFiles[filePath] = string(b)
  }
  mapValue, diags := types.MapValueFrom(ctx, types.StringType, repositoryFiles)
  resp.Diagnostics.Append(diags...)
@@ -469,8 +472,6 @@ func (r *bootstrapGitResource) Create(ctx context.Context, req resource.CreateRe
  resp.Diagnostics.Append(diags...)
 }
 
-// TODO: Consider if more value reading should be done here to detect drift. Similar to how import works.
-// TODO: Resources in the cluster should be verified to exist. If not resource id should be set to nil. This is to detect changing clusters.
 func (r *bootstrapGitResource) Read(ctx context.Context, req resource.ReadRequest, resp *resource.ReadResponse) {
  if r.prd == nil {
  resp.Diagnostics.AddError(missingConfiguration, bootstrapGitResourceMissingConfigError)
@@ -498,15 +499,15 @@ func (r *bootstrapGitResource) Read(ctx context.Context, req resource.ReadReques
  }
  defer os.RemoveAll(gitClient.Path())
 
- // Sync git repository with Terraform state.
+ // Detect drift for the Flux manifests stored in Git.
  repositoryFiles := map[string]string{}
  for k := range data.RepositoryFiles.Elements() {
- path := filepath.Join(gitClient.Path(), k)
- if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
- tflog.Debug(ctx, "Skip reading file that no longer exists in git repository", map[string]interface{}{"path": path})
+ filePath := filepath.Join(gitClient.Path(), k)
+ if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) {
+ tflog.Debug(ctx, "Skip reading file that no longer exists in git repository", map[string]interface{}{"path": filePath})
  continue
  }
- b, err := os.ReadFile(path)
+ b, err := os.ReadFile(filePath)
  if err != nil {
  resp.Diagnostics.AddError("Unable to read file in git repository", err.Error())
  return
@@ -520,11 +521,27 @@ func (r *bootstrapGitResource) Read(ctx context.Context, req resource.ReadReques
  }
  data.RepositoryFiles = mapValue
 
+ kubeClient, err := r.prd.GetKubernetesClient()
+ if err != nil {
+ resp.Diagnostics.AddError("Kubernetes Client", err.Error())
+ return
+ }
+
+ // Check cluster access and kubeconfig permissions
+ if err := isKubernetesReady(ctx, kubeClient); err != nil {
+ resp.Diagnostics.AddError("Kubernetes cluster", err.Error())
+ return
+ }
+
+ // TODO:Figure out how to detect drift for the Flux installation in the cluster.
+ //if !isFluxReady(ctx, kubeClient, data) {
+ // data.ID = types.StringNull()
+ //}
+
  diags = resp.State.Set(ctx, &data)
  resp.Diagnostics.Append(diags...)
 }
 
-// TODO: Verify Flux components after updating Git.
 func (r bootstrapGitResource) Update(ctx context.Context, req resource.UpdateRequest, resp *resource.UpdateResponse) {
  if r.prd == nil {
  resp.Diagnostics.AddError(missingConfiguration, bootstrapGitResourceMissingConfigError)
@@ -545,6 +562,46 @@ func (r bootstrapGitResource) Update(ctx context.Context, req resource.UpdateReq
  ctx, cancel := context.WithTimeout(ctx, timeout)
  defer cancel()
 
+ kubeClient, err := r.prd.GetKubernetesClient()
+ if err != nil {
+ resp.Diagnostics.AddError("Kubernetes Client", err.Error())
+ return
+ }
+
+ gitClient, err := r.prd.GetGitClient(ctx)
+ if err != nil {
+ resp.Diagnostics.AddError("Git Client", err.Error())
+ return
+ }
+ defer os.RemoveAll(gitClient.Path())
+
+ installOpts := getInstallOptions(data)
+ syncOpts := getSyncOptions(data, r.prd.GetRepositoryURL(), r.prd.git.Branch.ValueString())
+ var secretOpts sourcesecret.Options
+ if data.DisableSecretCreation.ValueBool() {
+ secretOpts = sourcesecret.Options{
+ Name: data.SecretName.ValueString(),
+ Namespace: data.Namespace.ValueString(),
+ }
+ } else {
+ secretOpts, err = r.prd.GetSecretOptions(data.SecretName.ValueString(), data.Namespace.ValueString(), data.Path.ValueString())
+ if err != nil {
+ resp.Diagnostics.AddError("Could not get secret options", err.Error())
+ return
+ }
+ }
+
+ bootstrapOpts, err := r.prd.GetBootstrapOptions()
+ if err != nil {
+ resp.Diagnostics.AddError("Could not get bootstrap options", err.Error())
+ return
+ }
+ bootstrapProvider, err := bootstrap.NewPlainGitProvider(gitClient, kubeClient, bootstrapOpts...)
+ if err != nil {
+ resp.Diagnostics.AddError("Could not create bootstrap provider", err.Error())
+ return
+ }
+
  previousRepositoryFiles := types.MapNull(types.StringType)
  diags = req.State.GetAttribute(ctx, path.Root("repository_files"), &previousRepositoryFiles)
  resp.Diagnostics.Append(diags...)
@@ -558,29 +615,24 @@ func (r bootstrapGitResource) Update(ctx context.Context, req resource.UpdateReq
  return
  }
 
- err := retry.RetryContext(ctx, timeout, func() *retry.RetryError {
- gitClient, err := r.prd.GetGitClient(ctx)
- if err != nil {
- return retry.NonRetryableError(err)
- }
- defer os.RemoveAll(gitClient.Path())
-
+ // Sync Git repository with Terraform state.
+ err = retry.RetryContext(ctx, timeout, func() *retry.RetryError {
  // Files should be removed if they are present in the state but not the plan.
  for k := range previousRepositoryFiles.Elements() {
  _, ok := data.RepositoryFiles.Elements()[k]
  if ok {
  continue
  }
- path := filepath.Join(gitClient.Path(), k)
- _, err := os.Stat(path)
+ filePath := filepath.Join(gitClient.Path(), k)
+ _, err := os.Stat(filePath)
  if errors.Is(err, os.ErrNotExist) {
- tflog.Debug(ctx, "Skipping removing no longer tracked file as it does not exist", map[string]interface{}{"path": path})
+ tflog.Debug(ctx, "Skipping removing no longer tracked file as it does not exist", map[string]interface{}{"path": filePath})
  continue
  }
  if err != nil {
  retry.NonRetryableError(fmt.Errorf("could not stat no longer tracked file: %w", err))
  }
- err = os.Remove(path)
+ err = os.Remove(filePath)
  if err != nil {
  return retry.NonRetryableError(fmt.Errorf("could not remove no longer tracked file: %w", err))
  }
@@ -591,7 +643,7 @@ func (r bootstrapGitResource) Update(ctx context.Context, req resource.UpdateReq
  for k, v := range repositoryFiles {
  files[k] = strings.NewReader(v)
  }
- commit, signer, err := r.prd.CreateCommit("Update Flux")
+ commit, signer, err := r.prd.CreateCommit("Update Flux manifests")
  if err != nil {
  return retry.NonRetryableError(fmt.Errorf("unable to create commit: %w", err))
  }
@@ -613,6 +665,14 @@ func (r bootstrapGitResource) Update(ctx context.Context, req resource.UpdateReq
  resp.Diagnostics.AddError("Could not update Flux", err.Error())
  }
 
+ // Sync Flux installation with Git state.
+ manifestsBase := ""
+ err = bootstrap.Run(ctx, bootstrapProvider, manifestsBase, installOpts, secretOpts, syncOpts, 2*time.Second, timeout)
+ if err != nil {
+ resp.Diagnostics.AddError("Bootstrap run error", err.Error())
+ return
+ }
+
  diags = resp.State.Set(ctx, &data)
  resp.Diagnostics.Append(diags...)
 }
@@ -1046,3 +1106,25 @@ func getExpectedRepositoryFiles(data bootstrapGitResourceData, url *url.URL, bra
  repositoryFiles[filepath.Join(data.Path.ValueString(), data.Namespace.ValueString(), konfig.DefaultKustomizationFileName())] = getKustomizationFile(data)
  return repositoryFiles, nil
 }
+
+func isKubernetesReady(ctx context.Context, kubeClient client.Client) error {
+ var list apiextensionsv1.CustomResourceDefinitionList
+ selector := client.MatchingLabels{manifestgen.PartOfLabelKey: manifestgen.PartOfLabelValue}
+ if err := kubeClient.List(ctx, &list, client.InNamespace(""), selector); err != nil {
+ return err
+ }
+ return nil
+}
+
+func isFluxReady(ctx context.Context, kubeClient client.Client, data bootstrapGitResourceData) bool {
+ syncName := apitypes.NamespacedName{
+ Namespace: data.Namespace.ValueString(),
+ Name: data.Namespace.ValueString(),
+ }
+ var rootSync *kustomizev1.Kustomization
+ if err := kubeClient.Get(ctx, syncName, rootSync); err != nil {
+ return false
+ }
+
+ return conditions.IsReady(rootSync)
+}

internal/provider/resource_bootstrap_git_test.go

@@ -99,7 +99,7 @@ kind: Kustomization
 resources:
  - gotk-components.yaml`
  env := environment{
- httpClone: "https://gitub.com",
+ httpClone: "https://git.example",
  }
 
  resource.ParallelTest(t, resource.TestCase{
@@ -115,7 +115,7 @@ resources:
 
 func TestAccBootstrapGit_TolerationKeys(t *testing.T) {
  env := environment{
- httpClone: "https://gitub.com",
+ httpClone: "https://git.example",
  }
  resource.ParallelTest(t, resource.TestCase{
  ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
@@ -217,15 +217,6 @@ func TestAccBootstrapGit_Drift(t *testing.T) {
  resource.TestCheckResourceAttrSet("flux_bootstrap_git.this", "repository_files.flux-system/gotk-sync.yaml"),
  ),
  },
- // Change path and expect files to be moved
- {
- Config: bootstrapGitCustomPath(env, "custom-path"),
- Check: resource.ComposeTestCheckFunc(
- resource.TestCheckResourceAttrSet("flux_bootstrap_git.this", "repository_files.custom-path/flux-system/kustomization.yaml"),
- resource.TestCheckResourceAttrSet("flux_bootstrap_git.this", "repository_files.custom-path/flux-system/gotk-components.yaml"),
- resource.TestCheckResourceAttrSet("flux_bootstrap_git.this", "repository_files.custom-path/flux-system/gotk-sync.yaml"),
- ),
- },
  },
  })
 }
@@ -497,28 +488,6 @@ EOF
  `, env.kubeCfgPath, env.sshClone, env.privateKey)
 }
 
-func bootstrapGitCustomPath(env environment, path string) string {
- return fmt.Sprintf(`
- provider "flux" {
- kubernetes = {
- config_path = "%s"
- }
- git = {
- url = "%s"
- http = {
- username = "%s"
- password = "%s"
- allow_insecure_http = true
- }
- }
- }
-
- resource "flux_bootstrap_git" "this" {
- path = "%s"
- }
- `, env.kubeCfgPath, env.httpClone, env.username, env.password, path)
-}
-
 func bootstrapGitVersion(env environment, version string) string {
  return fmt.Sprintf(`
  provider "flux" {