Implement drift detection and correction for cluster state

.github/workflows/golangci-lint.yml → .github/workflows/golangci-lint.yaml

@@ -1,4 +1,4 @@
-name: golangci-lint
+name: golangci
 on:
  pull_request:
  branches:
@@ -8,16 +8,25 @@ permissions:
  contents: read
 
 jobs:
- golangci-lint:
- name: golangci-lint
+ lint:
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
  - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
  with:
  go-version-file: 'go.mod'
  cache: true
- - name: golangci-lint
- uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0
+ - name: Run tidy
+ run: make tidy
+ - name: Check if working tree is dirty
+ run: |
+ if [[ $(git diff --stat) != '' ]]; then
+ git diff
+ echo 'run make tidy and commit changes'
+ exit 1
+ fi
+ - uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0
  with:
+ version: latest
+ skip-pkg-cache: true
  args: --timeout=10m

.github/workflows/tests.yaml

@@ -21,12 +21,7 @@ jobs:
  with:
  go-version-file: 'go.mod'
  cache: true
- - run: go mod download
- - run: go build -v .
- - name: Run linters
- uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0
- with:
- version: latest
+ - run: make build
  generate:
  runs-on: ubuntu-latest
  steps:
@@ -145,6 +140,31 @@ jobs:
  run: |
  flux check
  flux get all
+ - name: No-op apply Terraform
+ run: |
+ export TF_CLI_CONFIG_FILE="${PWD}/.terraformrc"
+ cd examples/github-via-ssh
+ terraform apply -auto-approve -var "github_token=${GITHUB_TOKEN}" -var "github_org=fluxcd-testing" -var "github_repository=${{ steps.vars.outputs.test_repo_name }}"
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }}
+ - name: Teardown Flux
+ run: |
+ flux uninstall -s --keep-namespace
+ kubectl delete ns flux-system
+ - name: Restore Flux with Terraform
+ run: |
+ export TF_CLI_CONFIG_FILE="${PWD}/.terraformrc"
+ cd examples/github-via-ssh
+ terraform apply -auto-approve -var "github_token=${GITHUB_TOKEN}" -var "github_org=fluxcd-testing" -var "github_repository=${{ steps.vars.outputs.test_repo_name }}"
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }}
+ - name: No-op apply Terraform
+ run: |
+ export TF_CLI_CONFIG_FILE="${PWD}/.terraformrc"
+ cd examples/github-via-ssh
+ terraform apply -auto-approve -var "github_token=${GITHUB_TOKEN}" -var "github_org=fluxcd-testing" -var "github_repository=${{ steps.vars.outputs.test_repo_name }}"
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }}
  - name: Destroy Terraform
  run: |
  cd examples/github-via-ssh

Makefile

@@ -22,6 +22,12 @@ test: tidy fmt vet
 testacc: tidy fmt vet
  TF_ACC=1 go test ./... -v -count $(TEST_COUNT) -parallel $(ACCTEST_PARALLELISM) -timeout $(ACCTEST_TIMEOUT)
 
+# Run acceptance tests on macOS with the gitea-flux instance
+# Requires the following entry in /etc/hosts:
+# 127.0.0.1 gitea-flux
+testmacos: tidy fmt vet
+ TF_ACC=1 GITEA_HOSTNAME=gitea-flux go test ./... -v -parallel 1 -run TestAccBootstrapGit_Drift
+
 build:
  CGO_ENABLED=0 go build -o ./bin/terraform-provider-flux main.go
 

docs/index.md

@@ -125,23 +125,23 @@ provider "flux" {
 
 ### Optional
 
-- `git` (Attributes) Configuration block with settings for Kubernetes. (see [below for nested schema](#nestedatt--git))
+- `git` (Attributes) Configuration block with settings for Git. (see [below for nested schema](#nestedatt--git))
 - `kubernetes` (Attributes) Configuration block with settings for Kubernetes. (see [below for nested schema](#nestedatt--kubernetes))
 
 <a id="nestedatt--git"></a>
 ### Nested Schema for `git`
 
 Required:
 
-- `url` (String) Url of git repository to bootstrap from.
+- `url` (String) Url of Git repository to bootstrap from.
 
 Optional:
 
 - `author_email` (String) Author email for Git commits.
 - `author_name` (String) Author name for Git commits. Defaults to `Flux`.
-- `branch` (String) Branch in repository to reconcile from. Defaults to `main`.
+- `branch` (String) Branch of the repository to reconcile from. Defaults to `main`.
 - `commit_message_appendix` (String) String to add to the commit messages.
-- `gpg_key_id` (String) Key id for selecting a particular key.
+- `gpg_key_id` (String) Key id for selecting a particular GPG key.
 - `gpg_key_ring` (String) Path to the GPG key ring for signing commits.
 - `gpg_passphrase` (String, Sensitive) Passphrase for decrypting GPG private key.
 - `http` (Attributes) (see [below for nested schema](#nestedatt--git--http))
@@ -163,7 +163,7 @@ Optional:
 
 Optional:
 
-- `password` (String, Sensitive) Password for private key.
+- `password` (String, Sensitive) Password of the SSH private key.
 - `private_key` (String, Sensitive) Private key used for authenticating to the Git SSH server.
 - `username` (String) Username for Git SSH server.
 

docs/resources/bootstrap_git.md

@@ -40,7 +40,7 @@ The following examples are available to help you use the provider:
 - `manifests_path` (String) The install manifests are built from a GitHub release or kustomize overlay if using a local path. Defaults to `https:/fluxcd/flux2/releases`.
 - `namespace` (String) The namespace scope for install manifests. Defaults to `flux-system`. It will be created if it does not exist.
 - `network_policy` (Boolean) Deny ingress access to the toolkit controllers from other namespaces using network policies. Defaults to `true`.
-- `path` (String) Path relative to the repository root, when specified the cluster sync will be scoped to this path.
+- `path` (String) Path relative to the repository root, when specified the cluster sync will be scoped to this path (immutable).
 - `recurse_submodules` (Boolean) Configures the GitRepository source to initialize and include Git submodules in the artifact it produces.
 - `registry` (String) Container registry where the toolkit images are published. Defaults to `ghcr.io/fluxcd`.
 - `secret_name` (String) Name of the secret the sync credentials can be found in or stored to. Defaults to `flux-system`.

go.mod

@@ -23,6 +23,7 @@ require (
  github.com/fluxcd/image-reflector-controller/api v0.31.2
  github.com/fluxcd/kustomize-controller/api v1.2.2
  github.com/fluxcd/notification-controller/api v1.2.4
+ github.com/fluxcd/pkg/apis/meta v1.4.0
  github.com/fluxcd/pkg/git v0.18.0
  github.com/fluxcd/pkg/git/gogit v0.18.0
  github.com/fluxcd/pkg/runtime v0.45.0
@@ -84,7 +85,6 @@ require (
  github.com/fluxcd/go-git-providers v0.20.0 // indirect
  github.com/fluxcd/pkg/apis/acl v0.2.0 // indirect
  github.com/fluxcd/pkg/apis/kustomize v1.3.0 // indirect
- github.com/fluxcd/pkg/apis/meta v1.4.0 // indirect
  github.com/fluxcd/pkg/kustomize v1.6.0 // indirect
  github.com/fluxcd/pkg/tar v0.4.0 // indirect
  github.com/fluxcd/pkg/version v0.3.0 // indirect
@@ -154,6 +154,7 @@ require (
  github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
  github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
  github.com/oklog/run v1.1.0 // indirect
+ github.com/onsi/gomega v1.32.0 // indirect
  github.com/opencontainers/go-digest v1.0.0 // indirect
  github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
  github.com/pelletier/go-toml v1.9.4 // indirect

go.sum

@@ -2,6 +2,8 @@ code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8=
 code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=

internal/provider/provider.go

@@ -207,18 +207,18 @@ func (p *fluxProvider) Schema(ctx context.Context, req provider.SchemaRequest, r
  Optional: true,
  },
  "git": schema.SingleNestedAttribute{
- Description: "Configuration block with settings for Kubernetes.",
+ Description: "Configuration block with settings for Git.",
  Attributes: map[string]schema.Attribute{
  "url": schema.StringAttribute{
  CustomType: customtypes.URLType{},
- Description: "Url of git repository to bootstrap from.",
+ Description: "Url of Git repository to bootstrap from.",
  Required: true,
  Validators: []validator.String{
  validators.URLScheme("http", "https", "ssh"),
  },
  },
  "branch": schema.StringAttribute{
- Description: fmt.Sprintf("Branch in repository to reconcile from. Defaults to `%s`.", defaultBranch),
+ Description: fmt.Sprintf("Branch of the repository to reconcile from. Defaults to `%s`.", defaultBranch),
  Optional: true,
  },
  "author_name": schema.StringAttribute{
@@ -239,7 +239,7 @@ func (p *fluxProvider) Schema(ctx context.Context, req provider.SchemaRequest, r
  Sensitive: true,
  },
  "gpg_key_id": schema.StringAttribute{
- Description: "Key id for selecting a particular key.",
+ Description: "Key id for selecting a particular GPG key.",
  Optional: true,
  },
  "commit_message_appendix": schema.StringAttribute{
@@ -253,7 +253,7 @@ func (p *fluxProvider) Schema(ctx context.Context, req provider.SchemaRequest, r
  Optional: true,
  },
  "password": schema.StringAttribute{
- Description: "Password for private key.",
+ Description: "Password of the SSH private key.",
  Optional: true,
  Sensitive: true,
  },

internal/provider/provider_resource_data.go

@@ -71,8 +71,7 @@ func (prd *providerResourceData) GetKubernetesClient() (client.WithWatch, error)
  return kubeClient, nil
 }
 
-func (prd *providerResourceData) GetGitClient(ctx context.Context) (*gogit.Client, error) {
- // Git configuration
+func (prd *providerResourceData) GetGitClient(tmpDir string) (*gogit.Client, error) {
  authOpts, err := getAuthOpts(prd.git)
  if err != nil {
  return nil, err
@@ -82,24 +81,52 @@ func (prd *providerResourceData) GetGitClient(ctx context.Context) (*gogit.Clien
  clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
  }
 
+ gitClient, err := gogit.NewClient(tmpDir, authOpts, clientOpts...)
+ if err != nil {
+ return nil, fmt.Errorf("could not create git client: %w", err)
+ }
+
+ return gitClient, nil
+}
+
+func (prd *providerResourceData) CloneRepository(ctx context.Context) (*gogit.Client, error) {
  tmpDir, err := manifestgen.MkdirTempAbs("", "flux-bootstrap-")
  if err != nil {
  return nil, fmt.Errorf("could not create temporary working directory for git repository: %w", err)
  }
- client, err := gogit.NewClient(tmpDir, authOpts, clientOpts...)
+ gitClient, err := prd.GetGitClient(tmpDir)
  if err != nil {
  return nil, fmt.Errorf("could not create git client: %w", err)
  }
  // TODO: Need to conditionally clone here. If repository is empty this will fail.
- _, err = client.Clone(ctx, prd.GetRepositoryURL().String(), repository.CloneConfig{
+ _, err = gitClient.Clone(ctx, prd.GetRepositoryURL().String(), repository.CloneConfig{
  CheckoutStrategy: repository.CheckoutStrategy{
  Branch: prd.git.Branch.ValueString(),
  },
  })
  if err != nil {
  return nil, fmt.Errorf("could not clone git repository: %w", err)
  }
- return client, nil
+ return gitClient, nil
+}
+
+func (prd *providerResourceData) GetBootstrapProvider(tmpDir string) (*bootstrap.PlainGitBootstrapper, error) {
+ gitClient, err := prd.GetGitClient(tmpDir)
+ if err != nil {
+ return nil, fmt.Errorf("could not create git client: %w", err)
+ }
+
+ kubeClient, err := prd.GetKubernetesClient()
+ if err != nil {
+ return nil, fmt.Errorf("could not get Kubernetes client: %w", err)
+ }
+
+ bootstrapOpts, err := prd.GetBootstrapOptions()
+ if err != nil {
+ return nil, fmt.Errorf("could not get bootstrap options: %w", err)
+ }
+
+ return bootstrap.NewPlainGitProvider(gitClient, kubeClient, bootstrapOpts...)
 }
 
 func (prd *providerResourceData) GetBootstrapOptions() ([]bootstrap.GitOption, error) {