DataDog future prep

adding MuteMonitorScope and UnmuteMonitor
updating mocks with gomock
use mergo to perform a better DeepEqual in AddOrUpdate()
import formatting
error handling for DeleteMonitors

cmd/root.go

@@ -24,14 +24,15 @@ import (
  "syscall"
  "time"
 
- "github.com/fairwindsops/astro/pkg/controller"
- "github.com/fairwindsops/astro/pkg/kube"
- "github.com/fairwindsops/astro/pkg/metrics"
  "github.com/prometheus/client_golang/prometheus/promhttp"
  log "github.com/sirupsen/logrus"
  "github.com/spf13/cobra"
  "k8s.io/client-go/tools/leaderelection"
  "k8s.io/client-go/tools/leaderelection/resourcelock"
+
+ "github.com/fairwindsops/astro/pkg/controller"
+ "github.com/fairwindsops/astro/pkg/kube"
+ "github.com/fairwindsops/astro/pkg/metrics"
 )
 
 var (

go.mod

@@ -11,7 +11,7 @@ require (
  github.com/golang/mock v1.3.1
  github.com/gophercloud/gophercloud v0.7.0 // indirect
  github.com/hashicorp/golang-lru v0.5.1 // indirect
- github.com/imdario/mergo v0.3.7 // indirect
+ github.com/imdario/mergo v0.3.8
  github.com/onsi/gomega v1.5.0 // indirect
  github.com/prometheus/client_golang v0.9.4
  github.com/sirupsen/logrus v1.4.2

go.sum

@@ -86,8 +86,8 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
-github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
+github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -176,8 +176,6 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/zorkian/go-datadog-api v2.19.0+incompatible h1:G17NtfeHwrsQ2degevkwiVEirZENSOjbGOjLLT/kwa4=
-github.com/zorkian/go-datadog-api v2.19.0+incompatible/go.mod h1:PkXwHX9CUQa/FpB9ZwAD45N1uhCW4MT/Wj7m36PbKss=
 github.com/zorkian/go-datadog-api v2.25.0+incompatible h1:6uTEIky3RbhdqlZCdeGYBtIb+quwNwMdbYLADl+cASU=
 github.com/zorkian/go-datadog-api v2.25.0+incompatible/go.mod h1:PkXwHX9CUQa/FpB9ZwAD45N1uhCW4MT/Wj7m36PbKss=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

pkg/config/config.go

@@ -26,7 +26,7 @@ import (
  "time"
 
  log "github.com/sirupsen/logrus"
- datadog "github.com/zorkian/go-datadog-api"
+ ddapi "github.com/zorkian/go-datadog-api"
  "gopkg.in/yaml.v2"
 )
 
@@ -37,10 +37,10 @@ type ruleset struct {
 
 // A MonitorSet represents a collection of Monitors that applies to an object.
 type MonitorSet struct {
- ObjectType string  `yaml:"type"` // The type of object. Example: deployment
- Annotations []Annotation  `yaml:"match_annotations"` // Annotations an object must possess to be considered applicable for the monitors.
- BoundObjects []string  `yaml:"bound_objects,omitempty"` // A collection of ObjectTypes that are bound to the MonitorSet.
- Monitors map[string]datadog.Monitor `yaml:"monitors"` // A collection of Monitors.
+ ObjectType string `yaml:"type"` // The type of object. Example: deployment
+ Annotations []Annotation `yaml:"match_annotations"` // Annotations an object must possess to be considered applicable for the monitors.
+ BoundObjects []string `yaml:"bound_objects,omitempty"` // A collection of ObjectTypes that are bound to the MonitorSet.
+ Monitors map[string]ddapi.Monitor `yaml:"monitors"` // A collection of Monitors.
 }
 
 // An Annotation represent a kubernetes annotation.
@@ -59,13 +59,13 @@ type Event struct {
 
 // Config represents the application configuration.
 type Config struct {
- DatadogAPIKey string // datadog api key for the datadog account.
- DatadogAppKey string // datadog app key for the datadog account.
- ClusterName string // A unique name for the cluster.
- OwnerTag string // A unique tag to identify the owner of monitors.
- MonitorDefinitionsPath []string // A url or local path for the configuration file.
- Rulesets *ruleset // The collection of rulesets to manage.
- DryRun bool // when set to true monitors will not be managed in datadog.
+ DatadogAPIKey string // datadog api key for the datadog account
+ DatadogAppKey string // datadog app key for the datadog account
+ ClusterName string // A unique name for the cluster
+ OwnerTag string // A unique tag to identify the owner of monitors
+ MonitorDefinitionsPath []string // A url or local path for the configuration file
+ Rulesets *ruleset // The collection of rulesets to manage
+ DryRun bool // when set to true monitors will not be managed in datadog
 }
 
 // Override represents any datadog monitor fields annotations can be overridden
@@ -75,8 +75,8 @@ type Override struct {
 }
 
 // GetMatchingMonitors returns a collection of monitors that apply to the specified objectType and annotations.
-func (config *Config) GetMatchingMonitors(annotations map[string]string, objectType string, overrides map[string][]Override) *[]datadog.Monitor {
- var validMonitors []datadog.Monitor
+func (config *Config) GetMatchingMonitors(annotations map[string]string, objectType string, overrides map[string][]Override) *[]ddapi.Monitor {
+ var validMonitors []ddapi.Monitor
 
  for _, mSet := range *config.getMatchingRulesets(annotations, objectType, overrides) {
  for _, v := range mSet.Monitors {
@@ -136,8 +136,8 @@ func (config *Config) getMatchingRulesets(annotations map[string]string, objectT
 }
 
 // GetBoundMonitors returns a collection of monitors that are indirectly bound to objectTypes in the namespace specified.
-func (config *Config) GetBoundMonitors(nsAnnotations map[string]string, objectType string, overrides map[string][]Override) *[]datadog.Monitor {
- var linkedMonitors []datadog.Monitor
+func (config *Config) GetBoundMonitors(nsAnnotations map[string]string, objectType string, overrides map[string][]Override) *[]ddapi.Monitor {
+ var linkedMonitors []ddapi.Monitor
  mSets := config.getMatchingRulesets(nsAnnotations, "binding", overrides)
 
  for _, mSet := range *mSets {

pkg/config/config_test.go

@@ -5,7 +5,7 @@ import (
  "testing"
 
  "github.com/stretchr/testify/assert"
- datadog "github.com/zorkian/go-datadog-api"
+ ddapi "github.com/zorkian/go-datadog-api"
  corev1 "k8s.io/api/core/v1"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -76,7 +76,7 @@ func TestGetRulesetsValid(t *testing.T) {
  assert.Equal(t, items["title"], *mSet.Monitors[items["name"]].Name)
 
  monitors := cfg.GetMatchingMonitors(annotations, objectType, overrides)
- expected := []datadog.Monitor{}
+ var expected []ddapi.Monitor
  for _, value := range mSet.Monitors {
  expected = append(expected, value)
  }

pkg/controller/controller.go

@@ -19,9 +19,6 @@ import (
  "fmt"
  "time"
 
- "github.com/fairwindsops/astro/pkg/config"
- handler "github.com/fairwindsops/astro/pkg/handler"
- "github.com/fairwindsops/astro/pkg/kube"
  log "github.com/sirupsen/logrus"
  v1 "k8s.io/api/apps/v1"
  corev1 "k8s.io/api/core/v1"
@@ -33,6 +30,10 @@ import (
  "k8s.io/client-go/kubernetes"
  "k8s.io/client-go/tools/cache"
  "k8s.io/client-go/util/workqueue"
+
+ "github.com/fairwindsops/astro/pkg/config"
+ "github.com/fairwindsops/astro/pkg/handler"
+ "github.com/fairwindsops/astro/pkg/kube"
 )
 
 // KubeResourceWatcher contains the informer that watches Kubernetes objects and the queue that processes updates.

pkg/datadog/datadog.go

@@ -17,20 +17,24 @@ package datadog
 import (
  "errors"
  "reflect"
+ "sync"
+
+ "github.com/imdario/mergo"
+ log "github.com/sirupsen/logrus"
+ ddapi "github.com/zorkian/go-datadog-api"
 
  "github.com/fairwindsops/astro/pkg/config"
  "github.com/fairwindsops/astro/pkg/metrics"
- log "github.com/sirupsen/logrus"
- "github.com/zorkian/go-datadog-api"
- "sync"
 )
 
 // ClientAPI defines the interface for the Datadog client, for testing purposes
 type ClientAPI interface {
- GetMonitorsByMonitorTags(tags []string) ([]datadog.Monitor, error)
- CreateMonitor(*datadog.Monitor) (*datadog.Monitor, error)
- UpdateMonitor(*datadog.Monitor) error
+ CreateMonitor(*ddapi.Monitor) (*ddapi.Monitor, error)
  DeleteMonitor(id int) error
+ GetMonitorsByMonitorTags(tags []string) ([]ddapi.Monitor, error)
+ MuteMonitorScope(id int, muteMonitorScope *ddapi.MuteMonitorScope) error
+ UnmuteMonitor(id int) error
+ UpdateMonitor(*ddapi.Monitor) error
 }
 
 // DDMonitorManager is a higher-level wrapper around the Datadog API
@@ -44,18 +48,18 @@ var ddMonitorManagerInstance *DDMonitorManager
 // GetInstance returns a singleton DDMonitorManager, creating it if necessary
 func GetInstance() *DDMonitorManager {
  if ddMonitorManagerInstance == nil {
- config := config.GetInstance()
+ conf := config.GetInstance()
  ddMonitorManagerInstance = &DDMonitorManager{
- Datadog: datadog.NewClient(config.DatadogAPIKey, config.DatadogAppKey),
+ Datadog: ddapi.NewClient(conf.DatadogAPIKey, conf.DatadogAppKey),
  }
  }
  return ddMonitorManagerInstance
 }
 
 // AddOrUpdate will create a monitor if it doesn't exist or update one if it does.
 // It returns the Id of the monitor created or updated.
-func (ddman *DDMonitorManager) AddOrUpdate(monitor *datadog.Monitor) (*datadog.Monitor, error) {
- log.Infof("Update templated monitor:%v", *monitor.Name)
+func (ddman *DDMonitorManager) AddOrUpdate(monitor *ddapi.Monitor) (*ddapi.Monitor, error) {
+ log.Infof("Update templated monitor: %v", *monitor.Name)
  ddman.mux.Lock()
  defer ddman.mux.Unlock()
 
@@ -73,29 +77,28 @@ func (ddman *DDMonitorManager) AddOrUpdate(monitor *datadog.Monitor) (*datadog.M
  return provisioned, nil
  }
 
+ merged, err := mergeMonitors(*monitor, *ddMonitor)
+ if err != nil {
+ return nil, err
+ }
  //monitor exists
- if reflect.DeepEqual(monitor, ddMonitor) {
- log.Infof("Monitor %d exists and is up to date.", ddMonitor.Id)
+ if reflect.DeepEqual(*merged, *ddMonitor) {
+ log.Infof("Monitor %d exists and is up to date.", *ddMonitor.Id)
  } else {
  // monitor exists and needs updating.
- log.Infof("Monitor %d needs updating.", ddMonitor.Id)
-
- //TODO - do a deep merge of monitors
- updated := datadog.Monitor(*monitor)
- updated.Id = ddMonitor.Id
-
- err := ddman.Datadog.UpdateMonitor(&updated)
+ log.Infof("Monitor %d needs updating.", *ddMonitor.Id)
+ err := ddman.Datadog.UpdateMonitor(merged)
  if err != nil {
  metrics.DatadogErrCounter.Inc()
- log.Errorf("Could not update monitor %d: %s", ddMonitor.Id, err)
+ log.Errorf("Could not update monitor %d: %s", *ddMonitor.Id, err)
  return ddMonitor, err
  }
  }
  return ddMonitor, nil
 }
 
 // GetProvisionedMonitor returns a monitor with the same name from Datadog.
-func (ddman *DDMonitorManager) GetProvisionedMonitor(monitor *datadog.Monitor) (*datadog.Monitor, error) {
+func (ddman *DDMonitorManager) GetProvisionedMonitor(monitor *ddapi.Monitor) (*ddapi.Monitor, error) {
  monitors, err := ddman.GetProvisionedMonitors()
  if err != nil {
  metrics.DatadogErrCounter.Inc()
@@ -112,12 +115,12 @@ func (ddman *DDMonitorManager) GetProvisionedMonitor(monitor *datadog.Monitor) (
 }
 
 // GetProvisionedMonitors returns a collection of monitors managed by astro.
-func (ddman *DDMonitorManager) GetProvisionedMonitors() ([]datadog.Monitor, error) {
+func (ddman *DDMonitorManager) GetProvisionedMonitors() ([]ddapi.Monitor, error) {
  return ddman.Datadog.GetMonitorsByMonitorTags([]string{config.GetInstance().OwnerTag})
 }
 
 // DeleteMonitor deletes a monitor
-func (ddman *DDMonitorManager) DeleteMonitor(monitor *datadog.Monitor) error {
+func (ddman *DDMonitorManager) DeleteMonitor(monitor *ddapi.Monitor) error {
  ddMonitor, err := ddman.GetProvisionedMonitor(monitor)
  if err != nil {
  return ddman.Datadog.DeleteMonitor(*ddMonitor.Id)
@@ -132,13 +135,15 @@ func (ddman *DDMonitorManager) DeleteMonitors(tags []string) error {
  log.Infof("Deleting %d monitors.", len(monitors))
  if err != nil {
  metrics.DatadogErrCounter.Inc()
- log.Errorf("Error getting monitors: %v", err)
  return err
  }
 
  for _, ddMonitor := range monitors {
  log.Infof("Deleting monitor with id %d", *ddMonitor.Id)
- ddman.Datadog.DeleteMonitor(*ddMonitor.Id)
+ err := ddman.Datadog.DeleteMonitor(*ddMonitor.Id)
+ if err != nil {
+ return err
+ }
  }
  return nil
 }
@@ -168,8 +173,25 @@ func DeleteExtinctMonitors(monitors []string, tags []string) error {
  return nil
 }
 
+// mergeMonitors fills in zero/nil values in our proposed monitor with values that already exist from the DD API
+func mergeMonitors(newMon, baseMon ddapi.Monitor) (*ddapi.Monitor, error) {
+ err := mergo.Merge(newMon.Options, baseMon.Options)
+ if err != nil {
+ return &ddapi.Monitor{}, err
+ }
+ creator := ddapi.Creator{}
+ newMon.Creator = &creator
+ err = mergo.Merge(newMon.Creator, baseMon.Creator)
+ if err != nil {
+ return &ddapi.Monitor{}, err
+ }
+ newMon.OverallState = baseMon.OverallState
+ newMon.Id = baseMon.Id
+ return &newMon, nil
+}
+
 // contains returns a boolean indicating whether a collection of strings contains a monitor with the name of monitor item.
-func contains(collection []string, item datadog.Monitor) bool {
+func contains(collection []string, item ddapi.Monitor) bool {
  for _, name := range collection {
  if name == *item.Name {
  return true

pkg/handler/bound.go

@@ -17,12 +17,13 @@ package handler
 import (
  "fmt"
 
- "github.com/fairwindsops/astro/pkg/config"
- "github.com/fairwindsops/astro/pkg/kube"
  log "github.com/sirupsen/logrus"
  appsv1 "k8s.io/api/apps/v1"
  corev1 "k8s.io/api/core/v1"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/fairwindsops/astro/pkg/config"
+ "github.com/fairwindsops/astro/pkg/kube"
 )
 
 func updateBoundResources(namespace *corev1.Namespace, kc *kube.ClientInstance) {

pkg/handler/bound_test.go

@@ -3,14 +3,15 @@ package handler
 import (
  "testing"
 
- "github.com/fairwindsops/astro/pkg/config"
- "github.com/fairwindsops/astro/pkg/datadog"
- "github.com/fairwindsops/astro/pkg/kube"
  "github.com/golang/mock/gomock"
  "github.com/stretchr/testify/assert"
  appsv1 "k8s.io/api/apps/v1"
  corev1 "k8s.io/api/core/v1"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/fairwindsops/astro/pkg/config"
+ "github.com/fairwindsops/astro/pkg/datadog"
+ "github.com/fairwindsops/astro/pkg/kube"
 )
 
 func TestUpdateBoundResources(t *testing.T) {

pkg/handler/deployments.go

@@ -18,14 +18,15 @@ import (
  "fmt"
  "strings"
 
- "github.com/fairwindsops/astro/pkg/config"
- "github.com/fairwindsops/astro/pkg/datadog"
- "github.com/fairwindsops/astro/pkg/kube"
- "github.com/fairwindsops/astro/pkg/metrics"
  log "github.com/sirupsen/logrus"
  ddapi "github.com/zorkian/go-datadog-api"
  appsv1 "k8s.io/api/apps/v1"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/fairwindsops/astro/pkg/config"
+ "github.com/fairwindsops/astro/pkg/datadog"
+ "github.com/fairwindsops/astro/pkg/kube"
+ "github.com/fairwindsops/astro/pkg/metrics"
 )
 
 // OnDeploymentChanged is a handler that should be called when a deployment changes.

pkg/handler/deployments_test.go

@@ -3,13 +3,14 @@ package handler
 import (
  "testing"
 
- "github.com/fairwindsops/astro/pkg/config"
- "github.com/fairwindsops/astro/pkg/datadog"
- "github.com/fairwindsops/astro/pkg/kube"
  "github.com/golang/mock/gomock"
  appsv1 "k8s.io/api/apps/v1"
  corev1 "k8s.io/api/core/v1"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/fairwindsops/astro/pkg/config"
+ "github.com/fairwindsops/astro/pkg/datadog"
+ "github.com/fairwindsops/astro/pkg/kube"
 )
 
 func TestDeploymentChange(t *testing.T) {

pkg/handler/handler.go

@@ -21,11 +21,12 @@ import (
  "strings"
  "text/template"
 
- "github.com/fairwindsops/astro/pkg/config"
  log "github.com/sirupsen/logrus"
  ddapi "github.com/zorkian/go-datadog-api"
  appsv1 "k8s.io/api/apps/v1"
  corev1 "k8s.io/api/core/v1"
+
+ "github.com/fairwindsops/astro/pkg/config"
 )
 
 // OnUpdate is a handler that should be called when an object is updated.