Add flag to disable metrics and introspection

Adds the environment variables DISABLE_INTROSPECTION and DISABLE_METRICS
to make it possible to turn off metrics and debugging.

ipamd/introspect.go

@@ -16,20 +16,22 @@ package ipamd
 import (
  "encoding/json"
  "net/http"
+ "os"
  "strconv"
  "sync"
  "time"
 
- log "github.com/cihub/seelog"
- "github.com/prometheus/client_golang/prometheus/promhttp"
-
  "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"
  "github.com/aws/amazon-vpc-cni-k8s/pkg/utils"
+ log "github.com/cihub/seelog"
 )
 
 const (
- // IntrospectionPort is the port for ipamd introspection
- IntrospectionPort = 61678
+ // introspectionAddress is listening on localhost 61679 for ipamd introspection
+ introspectionAddress = "127.0.0.1:61679"
+
+ // Environment variable to disable the introspection endpoints
+ envDisableIntrospection = "DISABLE_INTROSPECTION"
 )
 
 type rootResponse struct {
@@ -42,29 +44,33 @@ type LoggingHandler struct {
 }
 
 func (lh LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- log.Info("Handling http request", "method", r.Method, "from", r.RemoteAddr, "uri", r.RequestURI)
+ log.Info("Handling http request: ", ", method: ", r.Method, ", from: ", r.RemoteAddr, ", URI: ", r.RequestURI)
  lh.h.ServeHTTP(w, r)
 }
 
-// SetupHTTP sets up ipamd introspection service endpoint
-func (c *IPAMContext) SetupHTTP() {
- server := c.setupServer()
+// ServeIntrospection sets up ipamd introspection endpoints
+func (c *IPAMContext) ServeIntrospection() {
+ if disableIntrospection() {
+ log.Info("Introspection endpoints disabled")
+ return
+ }
 
+ log.Info("Serving introspection endpoints on ", introspectionAddress)
+ server := c.setupIntrospectionServer()
  for {
  once := sync.Once{}
- utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error {
- // TODO, make this cancellable and use the passed in context; for
- // now, not critical if this gets interrupted
+ _ = utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error {
  err := server.ListenAndServe()
  once.Do(func() {
- log.Error("Error running http api", "err", err)
+ log.Error("Error running http API: ", err)
  })
  return err
  })
  }
 }
 
-func (c *IPAMContext) setupServer() *http.Server {
+func (c *IPAMContext) setupIntrospectionServer() *http.Server {
+ // If enabled, add introspection endpoints
  serverFunctions := map[string]func(w http.ResponseWriter, r *http.Request){
  "/v1/enis": eniV1RequestHandler(c),
  "/v1/eni-configs": eniConfigRequestHandler(c),
@@ -81,26 +87,24 @@ func (c *IPAMContext) setupServer() *http.Server {
  availableCommandResponse, err := json.Marshal(&availableCommands)
 
  if err != nil {
- log.Error("Failed to Marshal: %v", err)
+ log.Error("Failed to marshal: %v", err)
  }
 
  defaultHandler := func(w http.ResponseWriter, r *http.Request) {
  logErr(w.Write(availableCommandResponse))
  }
-
  serveMux := http.NewServeMux()
  serveMux.HandleFunc("/", defaultHandler)
  for key, fn := range serverFunctions {
  serveMux.HandleFunc(key, fn)
  }
- serveMux.Handle("/metrics", promhttp.Handler())
 
  // Log all requests and then pass through to serveMux
  loggingServeMux := http.NewServeMux()
  loggingServeMux.Handle("/", LoggingHandler{serveMux})
 
  server := &http.Server{
- Addr: ":" + strconv.Itoa(IntrospectionPort),
+ Addr: introspectionAddress,
  Handler: loggingServeMux,
  ReadTimeout: 5 * time.Second,
  WriteTimeout: 5 * time.Second,
@@ -112,7 +116,7 @@ func eniV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Requ
  return func(w http.ResponseWriter, r *http.Request) {
  responseJSON, err := json.Marshal(ipam.dataStore.GetENIInfos())
  if err != nil {
- log.Error("Failed to marshal ENI data: %v", err)
+ log.Errorf("Failed to marshal ENI data: %v", err)
  http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
  return
  }
@@ -124,7 +128,7 @@ func podV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Requ
  return func(w http.ResponseWriter, r *http.Request) {
  responseJSON, err := json.Marshal(ipam.dataStore.GetPodInfos())
  if err != nil {
- log.Error("Failed to marshal pod data: %v", err)
+ log.Errorf("Failed to marshal pod data: %v", err)
  http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
  return
  }
@@ -136,7 +140,7 @@ func eniConfigRequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.
  return func(w http.ResponseWriter, r *http.Request) {
  responseJSON, err := json.Marshal(ipam.eniConfig.Getter())
  if err != nil {
- log.Error("Failed to marshal ENI config: %v", err)
+ log.Errorf("Failed to marshal ENI config: %v", err)
  http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
  return
  }
@@ -148,7 +152,7 @@ func networkEnvV1RequestHandler() func(http.ResponseWriter, *http.Request) {
  return func(w http.ResponseWriter, r *http.Request) {
  responseJSON, err := json.Marshal(networkutils.GetConfigForDebug())
  if err != nil {
- log.Error("Failed to marshal network env var data: %v", err)
+ log.Errorf("Failed to marshal network env var data: %v", err)
  http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
  return
  }
@@ -160,7 +164,7 @@ func ipamdEnvV1RequestHandler() func(http.ResponseWriter, *http.Request) {
  return func(w http.ResponseWriter, r *http.Request) {
  responseJSON, err := json.Marshal(GetConfigForDebug())
  if err != nil {
- log.Error("Failed to marshal ipamd env var data: %v", err)
+ log.Errorf("Failed to marshal ipamd env var data: %v", err)
  http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
  return
  }
@@ -173,3 +177,19 @@ func logErr(_ int, err error) {
  log.Errorf("Write failed: %v", err)
  }
 }
+
+// disableIntrospection returns true if we should disable the introspection
+func disableIntrospection() bool {
+ return getEnvBoolWithDefault(envDisableIntrospection, false)
+}
+
+func getEnvBoolWithDefault(envName string, def bool) bool {
+ if strValue := os.Getenv(envName); strValue != "" {
+ parsedValue, err := strconv.ParseBool(strValue)
+ if err == nil {
+ return parsedValue
+ }
+ log.Errorf("Failed to parse %s, using default `%t`: %v", envName, def, err.Error())
+ }
+ return def
+}

ipamd/ipamd_test.go

@@ -42,7 +42,6 @@ const (
  primaryENIid = "eni-00000000"
  secENIid = "eni-00000001"
  testAttachmentID = "eni-00000000-attach"
- eniID = "eni-5731da78"
  primaryMAC = "12:ef:2a:98:e5:5a"
  secMAC = "12:ef:2a:98:e5:5b"
  primaryDevice = 0
@@ -51,10 +50,8 @@ const (
  secSubnet = "10.10.20.0/24"
  ipaddr01 = "10.10.10.11"
  ipaddr02 = "10.10.10.12"
- ipaddr03 = "10.10.10.13"
  ipaddr11 = "10.10.20.11"
  ipaddr12 = "10.10.20.12"
- ipaddr13 = "10.10.20.13"
  vpcCIDR = "10.10.0.0/16"
 )
 
@@ -163,12 +160,12 @@ func TestNodeInit(t *testing.T) {
 }
 
 func TestIncreaseIPPoolDefault(t *testing.T) {
- os.Unsetenv(envCustomNetworkCfg)
+ _ = os.Unsetenv(envCustomNetworkCfg)
  testIncreaseIPPool(t, false)
 }
 
 func TestIncreaseIPPoolCustomENI(t *testing.T) {
- os.Setenv(envCustomNetworkCfg, "true")
+ _ = os.Setenv(envCustomNetworkCfg, "true")
  testIncreaseIPPool(t, true)
 }
 
@@ -337,15 +334,15 @@ func TestGetWarmENITarget(t *testing.T) {
  ctrl, _, _, _, _, _ := setup(t)
  defer ctrl.Finish()
 
- os.Setenv("WARM_IP_TARGET", "5")
+ _ = os.Setenv("WARM_IP_TARGET", "5")
  warmIPTarget := getWarmIPTarget()
  assert.Equal(t, warmIPTarget, 5)
 
- os.Unsetenv("WARM_IP_TARGET")
+ _ = os.Unsetenv("WARM_IP_TARGET")
  warmIPTarget = getWarmIPTarget()
  assert.Equal(t, warmIPTarget, noWarmIPTarget)
 
- os.Setenv("WARM_IP_TARGET", "non-integer-string")
+ _ = os.Setenv("WARM_IP_TARGET", "non-integer-string")
  warmIPTarget = getWarmIPTarget()
  assert.Equal(t, warmIPTarget, noWarmIPTarget)
 }
@@ -355,32 +352,32 @@ func TestGetMaxENI(t *testing.T) {
  defer ctrl.Finish()
 
  // MaxENI 5 is less than upper bound of 10, so 5
- os.Setenv("MAX_ENI", "5")
+ _ = os.Setenv("MAX_ENI", "5")
  maxENI := getMaxENI(10)
  assert.Equal(t, maxENI, 5)
 
  // MaxENI 5 is greater than upper bound of 4, so 4
- os.Setenv("MAX_ENI", "5")
+ _ = os.Setenv("MAX_ENI", "5")
  maxENI = getMaxENI(4)
  assert.Equal(t, maxENI, 4)
 
  // MaxENI 0 is 0, which means disabled; so use upper bound
- os.Setenv("MAX_ENI", "0")
+ _ = os.Setenv("MAX_ENI", "0")
  maxENI = getMaxENI(4)
  assert.Equal(t, maxENI, 4)
 
  // MaxENI 1 is less than upper bound of 4, so 1.
- os.Setenv("MAX_ENI", "1")
+ _ = os.Setenv("MAX_ENI", "1")
  maxENI = getMaxENI(4)
  assert.Equal(t, maxENI, 1)
 
  // Empty MaxENI means disabled, so use upper bound
- os.Unsetenv("MAX_ENI")
+ _ = os.Unsetenv("MAX_ENI")
  maxENI = getMaxENI(10)
  assert.Equal(t, maxENI, 10)
 
  // Invalid MaxENI means disabled, so use upper bound
- os.Setenv("MAX_ENI", "non-integer-string")
+ _ = os.Setenv("MAX_ENI", "non-integer-string")
  maxENI = getMaxENI(10)
  assert.Equal(t, maxENI, 10)
 }
@@ -398,30 +395,30 @@ func TestGetWarmIPTargetState(t *testing.T) {
 
  mockContext.dataStore = datastore.NewDataStore()
 
- os.Unsetenv("WARM_IP_TARGET")
+ _ = os.Unsetenv("WARM_IP_TARGET")
  _, _, warmIPTargetDefined := mockContext.ipTargetState()
  assert.False(t, warmIPTargetDefined)
 
- os.Setenv("WARM_IP_TARGET", "5")
+ _ = os.Setenv("WARM_IP_TARGET", "5")
  short, over, warmIPTargetDefined := mockContext.ipTargetState()
  assert.True(t, warmIPTargetDefined)
  assert.Equal(t, 5, short)
  assert.Equal(t, 0, over)
 
  // add 2 addresses to datastore
- mockContext.dataStore.AddENI("eni-1", 1, true)
- mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.1")
- mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.2")
+ _ = mockContext.dataStore.AddENI("eni-1", 1, true)
+ _ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.1")
+ _ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.2")
 
  short, over, warmIPTargetDefined = mockContext.ipTargetState()
  assert.True(t, warmIPTargetDefined)
  assert.Equal(t, 3, short)
  assert.Equal(t, 0, over)
 
  // add 3 more addresses to datastore
- mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.3")
- mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.4")
- mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.5")
+ _ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.3")
+ _ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.4")
+ _ = mockContext.dataStore.AddIPv4AddressFromStore("eni-1", "1.1.1.5")
 
  short, over, warmIPTargetDefined = mockContext.ipTargetState()
  assert.True(t, warmIPTargetDefined)

ipamd/metrics.go

@@ -0,0 +1,73 @@
+// Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+// http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package ipamd
+
+import (
+ "net/http"
+ "strconv"
+ "sync"
+ "time"
+
+ log "github.com/cihub/seelog"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+
+ "github.com/aws/amazon-vpc-cni-k8s/pkg/utils"
+)
+
+const (
+ // metricsPort is the port for prometheus metrics
+ metricsPort = 61678
+
+ // Environment variable to disable the metrics endpoint on 61678
+ envDisableMetrics = "DISABLE_METRICS"
+)
+
+// ServeMetrics sets up ipamd metrics and introspection endpoints
+func (c *IPAMContext) ServeMetrics() {
+ if disableMetrics() {
+ log.Info("Metrics endpoint disabled")
+ return
+ }
+
+ log.Info("Serving metrics on port ", metricsPort)
+ server := c.setupMetricsServer()
+ for {
+ once := sync.Once{}
+ _ = utils.RetryWithBackoff(utils.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error {
+ err := server.ListenAndServe()
+ once.Do(func() {
+ log.Error("Error running http API: ", err)
+ })
+ return err
+ })
+ }
+}
+
+func (c *IPAMContext) setupMetricsServer() *http.Server {
+ // Always add the metrics endpoint
+ serveMux := http.NewServeMux()
+ serveMux.Handle("/metrics", promhttp.Handler())
+ server := &http.Server{
+ Addr: ":" + strconv.Itoa(metricsPort),
+ Handler: serveMux,
+ ReadTimeout: 5 * time.Second,
+ WriteTimeout: 5 * time.Second,
+ }
+ return server
+}
+
+// disableMetrics returns true if we should disable metrics
+func disableMetrics() bool {
+ return getEnvBoolWithDefault(envDisableMetrics, false)
+}

ipamd/rpc_handler.go

@@ -31,7 +31,7 @@ import (
 )
 
 const (
- port = "127.0.0.1:50051"
+ ipamdgRPCaddress = "127.0.0.1:50051"
 )
 
 type server struct {
@@ -91,7 +91,7 @@ func (s *server) DelNetwork(ctx context.Context, in *pb.DelNetworkRequest) (*pb.
 
 // RunRPCHandler handles request from gRPC
 func (c *IPAMContext) RunRPCHandler() error {
- lis, err := net.Listen("tcp", port)
+ lis, err := net.Listen("tcp", ipamdgRPCaddress)
  if err != nil {
  log.Errorf("Failed to listen gRPC port: %v", err)
  return errors.Wrap(err, "ipamd: failed to listen to gRPC port")