Update CNI metrics

.gitignore

@@ -6,4 +6,5 @@ verify-network
 .idea/
 *.iml
 .DS_Store
+cni-metrics-helper/cni-metrics-helper
 portmap

Makefile

@@ -12,7 +12,7 @@
 # language governing permissions and limitations under the License.
 #
 
-.PHONY: build-linux clean docker docker-build lint unit-test vet download-portmap build-docker-test
+.PHONY: all build-linux clean docker docker-build lint unit-test vet download-portmap build-docker-test build-metrics docker-metrics metrics-unit-test docker-metrics-test docker-vet
 
 IMAGE ?= amazon/amazon-k8s-cni
 VERSION ?= $(shell git describe --tags --always --dirty)
@@ -42,8 +42,8 @@ build-linux:
  GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o aws-k8s-agent -ldflags "$(LDFLAGS)"
  GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o aws-cni -ldflags "$(LDFLAGS)" ./plugins/routed-eni/
 
-# Build docker image
-docker: 
+# Build CNI Docker image
+docker:
  @docker build --build-arg arch="$(ARCH)" -f scripts/dockerfiles/Dockerfile.release -t "$(IMAGE):$(VERSION)" .
  @echo "Built Docker image \"$(IMAGE):$(VERSION)\""
 
@@ -64,6 +64,27 @@ docker-unit-test: build-docker-test
  docker run -e GO111MODULE=on \
  amazon-k8s-cni-test:latest make unit-test
 
+# Build metrics
+build-metrics:
+ GOOS=linux GOARCH=$(ARCH) CGO_ENABLED=0 go build -o cni-metrics-helper/cni-metrics-helper cni-metrics-helper/cni-metrics-helper.go
+
+# Build metrics Docker image
+docker-metrics:
+ @docker build --build-arg arch="$(ARCH)" -f scripts/dockerfiles/Dockerfile.metrics -t "amazon/cni-metrics-helper:$(VERSION)" .
+ @echo "Built Docker image \"amazon/cni-metrics-helper:$(VERSION)\""
+
+metrics-unit-test:
+ GOOS=linux CGO_ENABLED=1 go test -v -cover -race -timeout 10s ./cni-metrics-helper/metrics/...
+
+docker-metrics-test:
+ docker run -v $(shell pwd):/usr/src/app/src/github.com/aws/amazon-vpc-cni-k8s \
+ --workdir=/usr/src/app/src/github.com/aws/amazon-vpc-cni-k8s \
+ --env GOPATH=/usr/src/app \
+ golang:1.10 make metrics-unit-test
+
+# Build both CNI and metrics helper
+all: docker docker-metrics
+
 # golint
 # To install: go get -u golang.org/x/lint/golint
 lint:
@@ -86,4 +107,5 @@ docker-vet: build-docker-test
 clean:
  rm -f aws-k8s-agent
  rm -f aws-cni
+ rm -f cni-metrics-helper/cni-metrics-helper
  rm -f portmap

cni-metrics-helper/cni-metrics-helper.go

@@ -0,0 +1,115 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+// http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/golang/glog"
+ "github.com/spf13/pflag"
+
+ "github.com/aws/amazon-vpc-cni-k8s/cni-metrics-helper/metrics"
+ "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
+ "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher"
+)
+
+type options struct {
+ kubeconfig string
+ pullInterval int
+ pullCNI bool
+ submitCW bool
+ help bool
+}
+
+func main() {
+ options := &options{}
+ flags := pflag.NewFlagSet("", pflag.ExitOnError)
+ // Add glog flags
+ flags.AddGoFlagSet(flag.CommandLine)
+ _ = flags.Lookup("logtostderr").Value.Set("true")
+ flags.Lookup("logtostderr").DefValue = "true"
+ flags.Lookup("logtostderr").NoOptDefVal = "true"
+ flags.BoolVar(&options.submitCW, "cloudwatch", true, "a bool")
+
+ flags.Usage = func() {
+ _, _ = fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
+ flags.PrintDefaults()
+ }
+
+ err := flags.Parse(os.Args)
+ if err != nil {
+ glog.Fatalf("Error on parsing parameters: %s", err)
+ }
+
+ err = flag.CommandLine.Parse([]string{})
+ if err != nil {
+ glog.Fatalf("Error on parsing commandline: %s", err)
+ }
+
+ if options.help {
+ flags.Usage()
+ os.Exit(1)
+ }
+
+ cwENV, found := os.LookupEnv("USE_CLOUDWATCH")
+ if found {
+ if strings.Compare(cwENV, "yes") == 0 {
+ options.submitCW = true
+ }
+ if strings.Compare(cwENV, "no") == 0 {
+ options.submitCW = false
+ }
+ }
+
+ glog.Infof("Starting CNIMetricsHelper, cloudwatch: %v...", options.submitCW)
+
+ kubeClient, err := k8sapi.CreateKubeClient()
+ if err != nil {
+ glog.Errorf("Failed to create client: %v", err)
+ os.Exit(1)
+ }
+
+ discoverController := k8sapi.NewController(kubeClient)
+ go discoverController.DiscoverK8SPods()
+
+ var cw publisher.Publisher
+
+ if options.submitCW {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ cw, err = publisher.New(ctx)
+ if err != nil {
+ glog.Errorf("Failed to create publisher: %v", err)
+ os.Exit(1)
+ }
+ go cw.Start()
+ defer cw.Stop()
+ }
+
+ var cniMetric *metrics.CNIMetricsTarget
+ cniMetric = metrics.CNIMetricsNew(kubeClient, cw, discoverController, options.submitCW)
+
+ // metric loop
+ var pullInterval = 30 // seconds
+ for range time.Tick(time.Duration(pullInterval) * time.Second) {
+ glog.Info("Collecting metrics ...")
+ metrics.Handler(cniMetric)
+ }
+}

cni-metrics-helper/metrics/cni_metrics.go

@@ -0,0 +1,167 @@
+// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+// http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+// Package metrics handles the processing of all metrics. This file handles metrics for ipamd
+package metrics
+
+import (
+ "github.com/golang/glog"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ clientset "k8s.io/client-go/kubernetes"
+
+ "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
+ "github.com/aws/amazon-vpc-cni-k8s/pkg/publisher"
+)
+
+// Port where prometheus metrics are published.
+const metricsPort = 61678
+
+// InterestingCNIMetrics defines metrics parsing definition for kube-state-metrics
+var InterestingCNIMetrics = map[string]metricsConvert{
+ "awscni_assigned_ip_addresses": {
+ actions: []metricsAction{
+ {cwMetricName: "assignIPAddresses",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{}}}},
+ "awscni_total_ip_addresses": {
+ actions: []metricsAction{
+ {cwMetricName: "totalIPAddresses",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{}}}},
+ "awscni_eni_allocated": {
+ actions: []metricsAction{
+ {cwMetricName: "eniAllocated",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{}}}},
+ "awscni_eni_max": {
+ actions: []metricsAction{
+ {cwMetricName: "eniMaxAvailable",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{}}}},
+ "awscni_ip_max": {
+ actions: []metricsAction{
+ {cwMetricName: "maxIPAddresses",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{}}}},
+ "awscni_aws_api_latency_ms": {
+ actions: []metricsAction{
+ {cwMetricName: "awsAPILatency",
+ matchFunc: matchAny,
+ actionFunc: metricsMax,
+ data: &dataPoints{},
+ logToFile: true}}},
+ "awscni_aws_api_error_count": {
+ actions: []metricsAction{
+ {cwMetricName: "awsAPIErr",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{},
+ logToFile: true}}},
+ "awscni_aws_utils_error_count": {
+ actions: []metricsAction{
+ {cwMetricName: "awsUtilErr",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{},
+ logToFile: true}}},
+ "awscni_ipamd_error_count": {
+ actions: []metricsAction{
+ {cwMetricName: "ipamdErr",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{},
+ logToFile: true}}},
+ "awscni_ipamd_action_inprogress": {
+ actions: []metricsAction{
+ {cwMetricName: "ipamdActionInProgress",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{},
+ logToFile: true}}},
+ "awscni_reconcile_count": {
+ actions: []metricsAction{
+ {cwMetricName: "reconcileCount",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{},
+ logToFile: true}}},
+ "awscni_add_ip_req_count": {
+ actions: []metricsAction{
+ {cwMetricName: "addReqCount",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{},
+ logToFile: true}}},
+ "awscni_del_ip_req_count": {
+ actions: []metricsAction{
+ {cwMetricName: "delReqCount",
+ matchFunc: matchAny,
+ actionFunc: metricsAdd,
+ data: &dataPoints{},
+ logToFile: true}}},
+}
+
+// CNIMetricsTarget defines data structure for kube-state-metric target
+type CNIMetricsTarget struct {
+ interestingMetrics map[string]metricsConvert
+ cwMetricsPublisher publisher.Publisher
+ kubeClient clientset.Interface
+ cniPods []string
+ discoveryController *k8sapi.Controller
+ submitCW bool
+}
+
+// CNIMetricsNew creates a new metricsTarget
+func CNIMetricsNew(c clientset.Interface, cw publisher.Publisher, d *k8sapi.Controller, submitCW bool) *CNIMetricsTarget {
+ return &CNIMetricsTarget{
+ interestingMetrics: InterestingCNIMetrics,
+ cwMetricsPublisher: cw,
+ kubeClient: c,
+ discoveryController: d,
+ submitCW: submitCW,
+ }
+}
+
+func (t *CNIMetricsTarget) grabMetricsFromTarget(cniPod string) ([]byte, error) {
+ glog.Infof("Grabbing metrics from CNI: %s", cniPod)
+ output, err := getMetricsFromPod(t.kubeClient, cniPod, metav1.NamespaceSystem, metricsPort)
+ if err != nil {
+ glog.Errorf("grabMetricsFromTarget: Failed to grab CNI endpoint: %v", err)
+ return nil, err
+ }
+
+ glog.V(5).Infof("cni-metrics text output: %s", string(output))
+ return output, nil
+}
+
+func (t *CNIMetricsTarget) getInterestingMetrics() map[string]metricsConvert {
+ return InterestingCNIMetrics
+}
+
+func (t *CNIMetricsTarget) getCWMetricsPublisher() publisher.Publisher {
+ return t.cwMetricsPublisher
+}
+
+func (t *CNIMetricsTarget) getTargetList() []string {
+ pods := t.discoveryController.GetCNIPods()
+ return pods
+}
+
+func (t *CNIMetricsTarget) submitCloudWatch() bool {
+ return t.submitCW
+}