Merge pull request #81 from bosh-prometheus/fix-tcp-stacking

Fix tcp stacking and concurrency handling

.github/workflows/main.yml → .github/workflows/goreleaser.yml

@@ -15,22 +15,14 @@ jobs:
  runs-on: ubuntu-latest
  steps:
  - name: checkout
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
  with:
  fetch-depth: 0
 
  - name: set up go
- uses: actions/setup-go@v2
+ uses: actions/setup-go@v4
  with:
- go-version: 1.18
-
- - name: cache go modules
- uses: actions/cache@v1
- with:
- path: ~/go/pkg/mod
- key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
- restore-keys: |
- ${{ runner.os }}-go-
+ go-version: ">=1.20"
 
  - name: tests modules
  run: |
@@ -51,13 +43,13 @@ jobs:
  go test -v ./...
 
  - name: Login to DockerHub
- uses: docker/login-action@v1
+ uses: docker/login-action@v2
  with:
  username: ${{ secrets.DOCKERHUB_USERNAME }}
  password: ${{ secrets.DOCKERHUB_TOKEN }}
 
  - name: run goreleaser
- uses: goreleaser/goreleaser-action@v2
+ uses: goreleaser/goreleaser-action@v4
  if: success() && startsWith(github.ref, 'refs/tags/')
  with:
  version: latest

.github/workflows/linter.yml

@@ -0,0 +1,22 @@
+name: linter
+
+on: push
+
+jobs:
+ linter:
+ runs-on: ubuntu-latest
+ steps:
+ - name: checkout
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: set up go
+ uses: actions/setup-go@v4
+ with:
+ go-version: ">=1.20"
+
+ - name: golangci-lint
+ uses: golangci/golangci-lint-action@v3
+ with:
+ args: --config .golangci.yml

.github/workflows/tests.yml

@@ -0,0 +1,22 @@
+name: tests
+
+
+on: push
+
+jobs:
+ tests:
+ runs-on: ubuntu-latest
+ steps:
+ - name: checkout
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+
+ - name: set up go
+ uses: actions/setup-go@v4
+ with:
+ go-version: ">=1.20"
+
+ - name: running unit-tests
+ run: |
+ go test -v ./...

.golangci.yml

@@ -0,0 +1,35 @@
+run:
+ timeout: 30m
+output:
+ format: line-number
+linters:
+ disable-all: false
+ enable:
+ - bodyclose
+ - depguard
+ - dogsled
+ #- dupl
+ - errcheck
+ - exportloopref
+ #- funlen
+ - gocognit
+ - goconst
+ - gocritic
+ - godox
+ - gofmt
+ - goimports
+ - gosec
+ - gosimple
+ - govet
+ - ineffassign
+ - misspell
+ - nakedret
+ - revive
+ - staticcheck
+ - stylecheck
+ - typecheck
+ - unconvert
+ - unparam
+ - unused
+ - whitespace
+ - reassign

Makefile

@@ -0,0 +1,12 @@
+all: test
+
+test:
+ @go test -v ./...
+
+check:
+ @golangci-lint run --config .golangci.yml
+
+coverage:
+ @go test -cover -coverprofile cover.out -v ./...
+ @go tool cover -func=cover.out
+ @rm -f cover.out

collectors/applications.go

@@ -10,11 +10,6 @@ import (
  log "github.com/sirupsen/logrus"
 )
 
-const (
- concurrentOrganizationsGoroutines = 10
- concurrentSpacesGoroutines = 10
-)
-
 type ApplicationsCollector struct {
  namespace string
  environment string
@@ -160,12 +155,11 @@ func NewApplicationsCollector(
 
 func (c ApplicationsCollector) Collect(objs *models.CFObjects, ch chan<- prometheus.Metric) {
  errorMetric := float64(0)
- err := objs.Error
  if objs.Error != nil {
  errorMetric = float64(1)
  c.applicationsScrapeErrorsTotalMetric.Inc()
  } else {
- err = c.reportApplicationsMetrics(objs, ch)
+ err := c.reportApplicationsMetrics(objs, ch)
  if err != nil {
  log.Error(err)
  errorMetric = float64(1)
@@ -198,12 +192,12 @@ func (c ApplicationsCollector) Describe(ch chan<- *prometheus.Desc) {
 }
 
 // reportApplicationsMetrics
-// 1. empty detected buildpacks for apps running on droplet
-// staged with a buildpack that is no mot available
-// fallback to buildpack field for compatibility with v0
-// 2. symmetrically in some corner cases, buildpack is null but
-// detected_buildpack is available. Use detected_buildpack
-// for compatibility with v0
+//  1. empty detected buildpacks for apps running on droplet
+//  staged with a buildpack that is no mot available
+//  fallback to buildpack field for compatibility with v0
+//  2. symmetrically in some corner cases, buildpack is null but
+//  detected_buildpack is available. Use detected_buildpack
+//  for compatibility with v0
 func (c ApplicationsCollector) reportApplicationsMetrics(objs *models.CFObjects, ch chan<- prometheus.Metric) error {
  c.applicationInfoMetric.Reset()
  c.applicationInstancesMetric.Reset()
@@ -223,21 +217,21 @@ func (c ApplicationsCollector) reportApplicationsMetrics(objs *models.CFObjects,
  }
  }
 
- spaceGuid, ok := application.Relationships[constant.RelationshipTypeSpace]
+ spaceRel, ok := application.Relationships[constant.RelationshipTypeSpace]
  if !ok {
  return fmt.Errorf("could not find space relation in application '%s'", application.GUID)
  }
- space, ok := objs.Spaces[spaceGuid.GUID]
+ space, ok := objs.Spaces[spaceRel.GUID]
  if !ok {
- return fmt.Errorf("could not find space with guid '%s'", spaceGuid.GUID)
+ return fmt.Errorf("could not find space with guid '%s'", spaceRel.GUID)
  }
- orgGuid, ok := space.Relationships[constant.RelationshipTypeOrganization]
+ orgRel, ok := space.Relationships[constant.RelationshipTypeOrganization]
  if !ok {
  return fmt.Errorf("could not find org relation in space '%s'", space.GUID)
  }
- organization, ok := objs.Orgs[orgGuid.GUID]
+ organization, ok := objs.Orgs[orgRel.GUID]
  if !ok {
- return fmt.Errorf("could not find org with guid '%s'", orgGuid.GUID)
+ return fmt.Errorf("could not find org with guid '%s'", orgRel.GUID)
  }
  appSum, ok := objs.AppSummaries[application.GUID]
  if !ok {

collectors/buildpacks.go

@@ -2,8 +2,9 @@ package collectors
 
 import (
  "time"
- "github.com/prometheus/client_golang/prometheus"
+
  "github.com/bosh-prometheus/cf_exporter/models"
+ "github.com/prometheus/client_golang/prometheus"
 )
 
 type BuildpacksCollector struct {

collectors/collectors.go

@@ -1,9 +1,9 @@
 package collectors
 
 import (
- "github.com/bosh-prometheus/cf_exporter/models"
- "github.com/bosh-prometheus/cf_exporter/filters"
  "github.com/bosh-prometheus/cf_exporter/fetcher"
+ "github.com/bosh-prometheus/cf_exporter/filters"
+ "github.com/bosh-prometheus/cf_exporter/models"
  "github.com/prometheus/client_golang/prometheus"
 )
 
@@ -13,22 +13,24 @@ type ObjectCollector interface {
 }
 
 type Collector struct {
- fetcher *fetcher.Fetcher
- filter *filters.Filter
+ workers int
+ config *fetcher.CFConfig
+ filter *filters.Filter
  collectors []ObjectCollector
 }
 
 func NewCollector(
  namespace string,
  environment string,
  deployment string,
- fetcher *fetcher.Fetcher,
+ workers int,
+ config *fetcher.CFConfig,
  filter *filters.Filter,
 ) (*Collector, error) {
-
  res := &Collector{
- fetcher: fetcher,
- filter: filter,
+ workers: workers,
+ config: config,
+ filter: filter,
  collectors: []ObjectCollector{},
  }
 
@@ -100,9 +102,9 @@ func NewCollector(
  return res, nil
 }
 
-
 func (c *Collector) Collect(ch chan<- prometheus.Metric) {
- objs := c.fetcher.GetObjects()
+ fetcher := fetcher.NewFetcher(c.workers, c.config, c.filter)
+ objs := fetcher.GetObjects()
  for _, collector := range c.collectors {
  collector.Collect(objs, ch)
  }

collectors/events.go

@@ -5,7 +5,6 @@ import (
 
  "github.com/bosh-prometheus/cf_exporter/models"
  "github.com/prometheus/client_golang/prometheus"
- log "github.com/sirupsen/logrus"
 )
 
 type EventsCollector struct {
@@ -18,8 +17,8 @@ type EventsCollector struct {
  lastEventsScrapeErrorMetric prometheus.Gauge
  lastEventsScrapeTimestampMetric prometheus.Gauge
  lastEventsScrapeDurationSecondsMetric prometheus.Gauge
- lastCheckFilter time.Time
- timeLocation *time.Location
+ lastCheckFilter  time.Time
+ timeLocation  *time.Location
 }
 
 func NewEventsCollector(
@@ -108,17 +107,11 @@ func NewEventsCollector(
 
 func (c *EventsCollector) Collect(objs *models.CFObjects, ch chan<- prometheus.Metric) {
  errorMetric := float64(0)
- err := objs.Error
  if objs.Error != nil {
  errorMetric = float64(1)
  c.eventsScrapeErrorsTotalMetric.Inc()
  } else {
- err = c.reportEventsMetrics(objs, ch)
- if err != nil {
- log.Error(err)
- errorMetric = float64(1)
- c.eventsScrapeErrorsTotalMetric.Inc()
- }
+ c.reportEventsMetrics(objs, ch)
  }
 
  c.eventsScrapeErrorsTotalMetric.Collect(ch)
@@ -143,7 +136,7 @@ func (c *EventsCollector) Describe(ch chan<- *prometheus.Desc) {
 
 // reportEventsMetrics
 // 1. find user's username in user map if available
-func (c *EventsCollector) reportEventsMetrics(objs *models.CFObjects, ch chan<- prometheus.Metric) error {
+func (c *EventsCollector) reportEventsMetrics(objs *models.CFObjects, ch chan<- prometheus.Metric) {
  c.eventsInfoMetric.Reset()
 
  for _, event := range objs.Events {
@@ -174,5 +167,4 @@ func (c *EventsCollector) reportEventsMetrics(objs *models.CFObjects, ch chan<-
  timeLocation, _ := time.LoadLocation("UTC")
  c.lastCheckFilter = time.Now().In(timeLocation)
  c.eventsInfoMetric.Collect(ch)
- return nil
 }

collectors/isolation_segments.go

@@ -2,8 +2,9 @@ package collectors
 
 import (
  "time"
- "github.com/prometheus/client_golang/prometheus"
+
  "github.com/bosh-prometheus/cf_exporter/models"
+ "github.com/prometheus/client_golang/prometheus"
 )
 
 type IsolationSegmentsCollector struct {

collectors/organizations.go

@@ -232,12 +232,11 @@ func NewOrganizationsCollector(
 
 func (c OrganizationsCollector) Collect(objs *models.CFObjects, ch chan<- prometheus.Metric) {
  errorMetric := float64(0)
- err := objs.Error
  if objs.Error != nil {
  errorMetric = float64(1)
  c.organizationsScrapeErrorsTotalMetric.Inc()
  } else {
- err = c.reportOrganizationsMetrics(objs, ch)
+ err := c.reportOrganizationsMetrics(objs, ch)
  if err != nil {
  log.Error(err)
  errorMetric = float64(1)

collectors/routes.go

@@ -5,7 +5,6 @@ import (
 
  "github.com/bosh-prometheus/cf_exporter/models"
  "github.com/prometheus/client_golang/prometheus"
- log "github.com/sirupsen/logrus"
 )
 
 type RoutesCollector struct {
@@ -101,17 +100,11 @@ func NewRoutesCollector(
 
 func (c RoutesCollector) Collect(objs *models.CFObjects, ch chan<- prometheus.Metric) {
  errorMetric := float64(0)
- err := objs.Error
  if objs.Error != nil {
  errorMetric = float64(1)
  c.routesScrapeErrorsTotalMetric.Inc()
  } else {
- err = c.reportRoutesMetrics(objs, ch)
- if err != nil {
- log.Error(err)
- errorMetric = float64(1)
- c.routesScrapeErrorsTotalMetric.Inc()
- }
+ c.reportRoutesMetrics(objs, ch)
  }
  c.routesScrapeErrorsTotalMetric.Collect(ch)
  c.routesScrapesTotalMetric.Inc()
@@ -133,7 +126,7 @@ func (c RoutesCollector) Describe(ch chan<- *prometheus.Desc) {
  c.lastRoutesScrapeDurationSecondsMetric.Describe(ch)
 }
 
-func (c RoutesCollector) reportRoutesMetrics(objs *models.CFObjects, ch chan<- prometheus.Metric) error {
+func (c RoutesCollector) reportRoutesMetrics(objs *models.CFObjects, ch chan<- prometheus.Metric) {
  c.routeInfoMetric.Reset()
 
  for _, route := range objs.Routes {
@@ -152,5 +145,4 @@ func (c RoutesCollector) reportRoutesMetrics(objs *models.CFObjects, ch chan<- p
  }
 
  c.routeInfoMetric.Collect(ch)
- return nil
 }