fix(pkg/stash): don't check status of all etcd endpoints on start

Athens checks the status of all etcd endpoints when started, which can cause
issues when some members of the etcd cluster are unavailable. It is perfectly
okay for some members of an etcd cluster to be unavailable, as it's designed
for high availability and fault tolerance.

The management of the connections is instead deferred to the etcd client, which
will handle failures and load balancing as expected.

Fixes: #1888

pkg/stash/with_etcd.go

@@ -10,39 +10,21 @@ import (
  "github.com/gomods/athens/pkg/storage"
  clientv3 "go.etcd.io/etcd/client/v3"
  "go.etcd.io/etcd/client/v3/concurrency"
- "golang.org/x/sync/errgroup"
 )
 
 // WithEtcd returns a distributed singleflight
 // using an etcd cluster. If it cannot connect,
 // to any of the endpoints, it will return an error.
 func WithEtcd(endpoints []string, checker storage.Checker) (Wrapper, error) {
- const op errors.Op = "stash.WithEtcd"
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
  c, err := clientv3.New(clientv3.Config{
  Endpoints: endpoints,
- DialTimeout: time.Second * 5,
+ DialTimeout: 5 * time.Second,
  })
  if err != nil {
- return nil, errors.E(op, err)
- }
- var eg errgroup.Group
- for _, ep := range endpoints {
- epStat := func(ep string) func() error {
- return func() error {
- _, err := c.Status(ctx, ep)
- return err
- }
- }(ep)
- eg.Go(epStat)
- }
- err = eg.Wait()
- if err != nil {
- return nil, errors.E(op, err)
+ return nil, errors.E("stash.WithEtcd", err)
  }
  return func(s Stasher) Stasher {
- return &etcd{c, s, checker}
+ return &etcd{client: c, stasher: s, checker: checker}
  }, nil
 }