Retry reconcile and clone actions once

We have observed that the code at times outperforms GitHub mechanics, resulting in not found errors that are only true for a millisecond. Retrying those actions once with a 2 second delay should be more friendly to users. Signed-off-by: Hidde Beydals <[email protected]>
fluxcd · Apr 7, 2021 · 7481c6b · 7481c6b
1 parent 4ece123
commit 7481c6b
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 8 deletions.
diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go
@@ -182,3 +182,17 @@ func kustomizationReconciled(ctx context.Context, kube client.Client, objKey cli
  return false, nil
  }
 }
+
+func retry(retries int, wait time.Duration, fn func() error) (err error) {
+ for i := 0; ; i++ {
+ err = fn()
+ if err == nil {
+ return
+ }
+ if i >= retries {
+ break
+ }
+ time.Sleep(wait)
+ }
+ return err
+}
diff --git a/internal/bootstrap/bootstrap_plain_git.go b/internal/bootstrap/bootstrap_plain_git.go
@@ -104,8 +104,11 @@ func (b *PlainGitBootstrapper) ReconcileComponents(ctx context.Context, manifest
  }
 
  b.logger.Actionf("cloning branch %q from Git repository %q", b.branch, b.url)
- cloned, err := b.git.Clone(ctx, b.url, b.branch)
- if err != nil {
+ var cloned bool
+ if err = retry(1, 2*time.Second, func() (err error) {
+ cloned, err = b.git.Clone(ctx, b.url, b.branch)
+ return
+ }); err != nil {
  return fmt.Errorf("failed to clone repository: %w", err)
  }
  if cloned {
@@ -216,12 +219,15 @@ func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options
  if _, err := b.git.Status(); err != nil {
  if err == git.ErrNoGitRepository {
  b.logger.Actionf("cloning branch %q from Git repository %q", b.branch, b.url)
- cloned, err := b.git.Clone(ctx, b.url, b.branch)
- if err != nil {
+ var cloned bool
+ if err = retry(1, 2*time.Second, func() (err error) {
+ cloned, err = b.git.Clone(ctx, b.url, b.branch)
+ return
+ }); err != nil {
  return fmt.Errorf("failed to clone repository: %w", err)
  }
  if cloned {
- b.logger.Successf("cloned repository", b.url)
+ b.logger.Successf("cloned repository")
  }
  }
  return err

diff --git a/internal/bootstrap/bootstrap_provider.go b/internal/bootstrap/bootstrap_provider.go
@@ -278,11 +278,15 @@ func (b *GitProviderBootstrapper) reconcileOrgRepository(ctx context.Context) (g
  }
  b.logger.Successf("repository %q created", repoRef.String())
  }
+
  // Set default branch before calling Reconcile due to bug described
  // above.
  repoInfo.DefaultBranch = repo.Get().DefaultBranch
  var changed bool
- if repo, changed, err = b.provider.OrgRepositories().Reconcile(ctx, repoRef, repoInfo); err != nil {
+ if err = retry(1, 2*time.Second, func() (err error) {
+ repo, changed, err = b.provider.OrgRepositories().Reconcile(ctx, repoRef, repoInfo)
+ return
+ }); err != nil {
  return nil, fmt.Errorf("failed to reconcile Git repository %q: %w", repoRef.String(), err)
  }
  if changed {
@@ -352,12 +356,16 @@ func (b *GitProviderBootstrapper) reconcileUserRepository(ctx context.Context) (
  // above.
  repoInfo.DefaultBranch = repo.Get().DefaultBranch
  var changed bool
- if repo, changed, err = b.provider.UserRepositories().Reconcile(ctx, repoRef, repoInfo); err != nil {
- return nil, err
+ if err = retry(1, 2*time.Second, func() (err error) {
+ repo, changed, err = b.provider.UserRepositories().Reconcile(ctx, repoRef, repoInfo)
+ return
+ }); err != nil {
+ return nil, fmt.Errorf("failed to reconcile Git repository %q: %w", repoRef.String(), err)
  }
  if changed {
  b.logger.Successf("repository %q reconciled", repoRef.String())
  }
+
  return repo, nil
 }