Refactor bootstrap to make use of go-git-providers

.github/workflows/bootstrap.yaml

@@ -47,7 +47,8 @@ jobs:
  --owner=fluxcd-testing \
  --repository=${{ steps.vars.outputs.test_repo_name }} \
  --branch=main \
- --path=test-cluster
+ --path=test-cluster \
+ --team=team-z
  env:
  GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }}
  - name: bootstrap no-op
@@ -56,7 +57,8 @@ jobs:
  --owner=fluxcd-testing \
  --repository=${{ steps.vars.outputs.test_repo_name }} \
  --branch=main \
- --path=test-cluster
+ --path=test-cluster \
+ --team=team-z
  env:
  GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }}
  - name: uninstall
@@ -69,7 +71,8 @@ jobs:
  --owner=fluxcd-testing \
  --repository=${{ steps.vars.outputs.test_repo_name }} \
  --branch=main \
- --path=test-cluster
+ --path=test-cluster \
+ --team=team-z
  env:
  GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }}
  - name: delete repository

cmd/flux/bootstrap.go

@@ -17,26 +17,15 @@ limitations under the License.
 package main
 
 import (
- "context"
+ "crypto/elliptic"
  "fmt"
- "path/filepath"
- "time"
+ "io/ioutil"
 
  "github.com/spf13/cobra"
- corev1 "k8s.io/api/core/v1"
- "k8s.io/apimachinery/pkg/types"
- "k8s.io/apimachinery/pkg/util/wait"
- "sigs.k8s.io/controller-runtime/pkg/client"
-
- kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
- sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
 
  "github.com/fluxcd/flux2/internal/flags"
  "github.com/fluxcd/flux2/internal/utils"
- "github.com/fluxcd/flux2/pkg/manifestgen/install"
- kus "github.com/fluxcd/flux2/pkg/manifestgen/kustomization"
- "github.com/fluxcd/flux2/pkg/manifestgen/sync"
- "github.com/fluxcd/flux2/pkg/status"
+ "github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret"
 )
 
 var bootstrapCmd = &cobra.Command{
@@ -46,21 +35,38 @@ var bootstrapCmd = &cobra.Command{
 }
 
 type bootstrapFlags struct {
- version string
+ version string
+ arch flags.Arch
+ logLevel flags.LogLevel
+
+ branch string
+ manifestsPath string
+
  defaultComponents []string
  extraComponents []string
- registry string
- imagePullSecret string
- branch string
+ requiredComponents []string
+
+ registry string
+ imagePullSecret string
+
+ secretName string
+ tokenAuth bool
+ keyAlgorithm flags.PublicKeyAlgorithm
+ keyRSABits flags.RSAKeyBits
+ keyECDSACurve flags.ECDSACurve
+ sshHostname string
+ caFile string
+ privateKeyFile string
+
  watchAllNamespaces bool
  networkPolicy bool
- manifestsPath string
- arch flags.Arch
- logLevel flags.LogLevel
- requiredComponents []string
- tokenAuth bool
  clusterDomain string
  tolerationKeys []string
+
+ authorName string
+ authorEmail string
+
+ commitMessageAppendix string
 }
 
 const (
@@ -72,232 +78,99 @@ var bootstrapArgs = NewBootstrapFlags()
 func init() {
  bootstrapCmd.PersistentFlags().StringVarP(&bootstrapArgs.version, "version", "v", "",
  "toolkit version, when specified the manifests are downloaded from https:/fluxcd/flux2/releases")
+
  bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapArgs.defaultComponents, "components", rootArgs.defaults.Components,
  "list of components, accepts comma-separated values")
  bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapArgs.extraComponents, "components-extra", nil,
  "list of components in addition to those supplied or defaulted, accepts comma-separated values")
+
  bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.registry, "registry", "ghcr.io/fluxcd",
  "container registry where the toolkit images are published")
  bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.imagePullSecret, "image-pull-secret", "",
  "Kubernetes secret name used for pulling the toolkit images from a private registry")
- bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.arch, "arch", bootstrapArgs.arch.Description())
+
  bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.branch, "branch", bootstrapDefaultBranch,
  "default branch (for GitHub this must match the default branch setting for the organization)")
+ bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.manifestsPath, "manifests", "", "path to the manifest directory")
+
  bootstrapCmd.PersistentFlags().BoolVar(&bootstrapArgs.watchAllNamespaces, "watch-all-namespaces", true,
  "watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed")
  bootstrapCmd.PersistentFlags().BoolVar(&bootstrapArgs.networkPolicy, "network-policy", true,
  "deny ingress access to the toolkit controllers from other namespaces using network policies")
  bootstrapCmd.PersistentFlags().BoolVar(&bootstrapArgs.tokenAuth, "token-auth", false,
  "when enabled, the personal access token will be used instead of SSH deploy key")
  bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.logLevel, "log-level", bootstrapArgs.logLevel.Description())
- bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.manifestsPath, "manifests", "", "path to the manifest directory")
  bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.clusterDomain, "cluster-domain", rootArgs.defaults.ClusterDomain, "internal cluster domain")
  bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapArgs.tolerationKeys, "toleration-keys", nil,
  "list of toleration keys used to schedule the components pods onto nodes with matching taints")
- bootstrapCmd.PersistentFlags().MarkHidden("manifests")
+
+ bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.secretName, "secret-name", rootArgs.defaults.Namespace, "name of the secret the sync credentials can be found in or stored to")
+ bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.keyAlgorithm, "ssh-key-algorithm", bootstrapArgs.keyAlgorithm.Description())
+ bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.keyRSABits, "ssh-rsa-bits", bootstrapArgs.keyRSABits.Description())
+ bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.keyECDSACurve, "ssh-ecdsa-curve", bootstrapArgs.keyECDSACurve.Description())
+ bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.sshHostname, "ssh-hostname", "", "SSH hostname, to be used when the SSH host differs from the HTTPS one")
+ bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.caFile, "ca-file", "", "path to TLS CA file used for validating self-signed certificates")
+ bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.privateKeyFile, "private-key-file", "", "path to a private key file used for authenticating to the Git SSH server")
+
+ bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.authorName, "author-name", "Flux", "author name for Git commits")
+ bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.authorEmail, "author-email", "", "author email for Git commits")
+
+ bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.commitMessageAppendix, "commit-message-appendix", "", "string to add to the commit messages, e.g. '[ci skip]'")
+
+ bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.arch, "arch", bootstrapArgs.arch.Description())
  bootstrapCmd.PersistentFlags().MarkDeprecated("arch", "multi-arch container image is now available for AMD64, ARMv7 and ARM64")
+ bootstrapCmd.PersistentFlags().MarkHidden("manifests")
+
  rootCmd.AddCommand(bootstrapCmd)
 }
 
 func NewBootstrapFlags() bootstrapFlags {
  return bootstrapFlags{
  logLevel: flags.LogLevel(rootArgs.defaults.LogLevel),
  requiredComponents: []string{"source-controller", "kustomize-controller"},
+ keyAlgorithm: flags.PublicKeyAlgorithm(sourcesecret.RSAPrivateKeyAlgorithm),
+ keyRSABits: 2048,
+ keyECDSACurve: flags.ECDSACurve{Curve: elliptic.P384()},
  }
 }
 
 func bootstrapComponents() []string {
  return append(bootstrapArgs.defaultComponents, bootstrapArgs.extraComponents...)
 }
 
-func bootstrapValidate() error {
- components := bootstrapComponents()
- for _, component := range bootstrapArgs.requiredComponents {
- if !utils.ContainsItemString(components, component) {
- return fmt.Errorf("component %s is required", component)
- }
- }
-
- if err := utils.ValidateComponents(components); err != nil {
- return err
- }
-
- return nil
-}
-
-func generateInstallManifests(targetPath, namespace, tmpDir string, localManifests string) (string, error) {
- if ver, err := getVersion(bootstrapArgs.version); err != nil {
- return "", err
- } else {
- bootstrapArgs.version = ver
- }
-
- manifestsBase := ""
- if isEmbeddedVersion(bootstrapArgs.version) {
- if err := writeEmbeddedManifests(tmpDir); err != nil {
- return "", err
- }
- manifestsBase = tmpDir
- }
-
- opts := install.Options{
- BaseURL: localManifests,
- Version: bootstrapArgs.version,
- Namespace: namespace,
- Components: bootstrapComponents(),
- Registry: bootstrapArgs.registry,
- ImagePullSecret: bootstrapArgs.imagePullSecret,
- WatchAllNamespaces: bootstrapArgs.watchAllNamespaces,
- NetworkPolicy: bootstrapArgs.networkPolicy,
- LogLevel: bootstrapArgs.logLevel.String(),
- NotificationController: rootArgs.defaults.NotificationController,
- ManifestFile: rootArgs.defaults.ManifestFile,
- Timeout: rootArgs.timeout,
- TargetPath: targetPath,
- ClusterDomain: bootstrapArgs.clusterDomain,
- TolerationKeys: bootstrapArgs.tolerationKeys,
- }
-
- if localManifests == "" {
- opts.BaseURL = rootArgs.defaults.BaseURL
- }
-
- output, err := install.Generate(opts, manifestsBase)
- if err != nil {
- return "", fmt.Errorf("generating install manifests failed: %w", err)
- }
-
- filePath, err := output.WriteFile(tmpDir)
- if err != nil {
- return "", fmt.Errorf("generating install manifests failed: %w", err)
- }
- return filePath, nil
-}
-
-func applyInstallManifests(ctx context.Context, manifestPath string, components []string) error {
- kubectlArgs := []string{"apply", "-f", manifestPath}
- if _, err := utils.ExecKubectlCommand(ctx, utils.ModeOS, rootArgs.kubeconfig, rootArgs.kubecontext, kubectlArgs...); err != nil {
- return fmt.Errorf("install failed: %w", err)
- }
- kubeConfig, err := utils.KubeConfig(rootArgs.kubeconfig, rootArgs.kubecontext)
- if err != nil {
- return fmt.Errorf("install failed: %w", err)
- }
- statusChecker, err := status.NewStatusChecker(kubeConfig, time.Second, rootArgs.timeout, logger)
- if err != nil {
- return fmt.Errorf("install failed: %w", err)
- }
- componentRefs, err := buildComponentObjectRefs(components...)
- if err != nil {
- return fmt.Errorf("install failed: %w", err)
- }
- logger.Waitingf("verifying installation")
- if err := statusChecker.Assess(componentRefs...); err != nil {
- return fmt.Errorf("install failed")
- }
- return nil
-}
-
-func generateSyncManifests(url, branch, name, namespace, targetPath, tmpDir string, interval time.Duration) (string, error) {
- opts := sync.Options{
- Name: name,
- Namespace: namespace,
- URL: url,
- Branch: branch,
- Interval: interval,
- Secret: namespace,
- TargetPath: targetPath,
- ManifestFile: sync.MakeDefaultOptions().ManifestFile,
+func buildEmbeddedManifestBase() (string, error) {
+ if !isEmbeddedVersion(bootstrapArgs.version) {
+ return "", nil
  }
-
- manifest, err := sync.Generate(opts)
- if err != nil {
- return "", fmt.Errorf("generating install manifests failed: %w", err)
- }
-
- output, err := manifest.WriteFile(tmpDir)
+ tmpBaseDir, err := ioutil.TempDir("", "flux-manifests-")
  if err != nil {
  return "", err
  }
- outputDir := filepath.Dir(output)
-
- kusOpts := kus.MakeDefaultOptions()
- kusOpts.BaseDir = tmpDir
- kusOpts.TargetPath = filepath.Dir(manifest.Path)
-
- kustomization, err := kus.Generate(kusOpts)
- if err != nil {
+ if err := writeEmbeddedManifests(tmpBaseDir); err != nil {
  return "", err
  }
- if _, err = kustomization.WriteFile(tmpDir); err != nil {
- return "", err
- }
-
- return outputDir, nil
+ return tmpBaseDir, nil
 }
 
-func applySyncManifests(ctx context.Context, kubeClient client.Client, name, namespace, manifestsPath string) error {
- kubectlArgs := []string{"apply", "-k", manifestsPath}
- if _, err := utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, rootArgs.kubeconfig, rootArgs.kubecontext, kubectlArgs...); err != nil {
- return err
- }
-
- logger.Waitingf("waiting for cluster sync")
-
- var gitRepository sourcev1.GitRepository
- if err := wait.PollImmediate(rootArgs.pollInterval, rootArgs.timeout,
- isGitRepositoryReady(ctx, kubeClient, types.NamespacedName{Name: name, Namespace: namespace}, &gitRepository)); err != nil {
- return err
+func bootstrapValidate() error {
+ components := bootstrapComponents()
+ for _, component := range bootstrapArgs.requiredComponents {
+ if !utils.ContainsItemString(components, component) {
+ return fmt.Errorf("component %s is required", component)
+ }
  }
 
- var kustomization kustomizev1.Kustomization
- if err := wait.PollImmediate(rootArgs.pollInterval, rootArgs.timeout,
- isKustomizationReady(ctx, kubeClient, types.NamespacedName{Name: name, Namespace: namespace}, &kustomization)); err != nil {
+ if err := utils.ValidateComponents(components); err != nil {
  return err
  }
 
  return nil
 }
 
-func shouldInstallManifests(ctx context.Context, kubeClient client.Client, namespace string) bool {
- namespacedName := types.NamespacedName{
- Namespace: namespace,
- Name: namespace,
- }
- var kustomization kustomizev1.Kustomization
- if err := kubeClient.Get(ctx, namespacedName, &kustomization); err != nil {
- return true
- }
-
- return kustomization.Status.LastAppliedRevision == ""
-}
-
-func shouldCreateDeployKey(ctx context.Context, kubeClient client.Client, namespace string) bool {
- namespacedName := types.NamespacedName{
- Namespace: namespace,
- Name: namespace,
+func mapTeamSlice(s []string, defaultPermission string) map[string]string {
+ m := make(map[string]string, len(s))
+ for _, v := range s {
+ m[v] = defaultPermission
  }
-
- var existing corev1.Secret
- if err := kubeClient.Get(ctx, namespacedName, &existing); err != nil {
- return true
- }
- return false
-}
-
-func checkIfBootstrapPathDiffers(ctx context.Context, kubeClient client.Client, namespace string, path string) (string, bool) {
- namespacedName := types.NamespacedName{
- Name: namespace,
- Namespace: namespace,
- }
- var fluxSystemKustomization kustomizev1.Kustomization
- err := kubeClient.Get(ctx, namespacedName, &fluxSystemKustomization)
- if err != nil {
- return "", false
- }
- if fluxSystemKustomization.Spec.Path == path {
- return "", false
- }
-
- return fluxSystemKustomization.Spec.Path, true
+ return m
 }