Add support for recent PostgreSQL versions in Metricbeat (#24402) (#2…

…4444)

Add testing with with versions 10, 11, 12 and 13.
Add fields available since version 10 in the activity
metricset to give more details about the backend and its waits.
Since version 12, `pg_stat_database` includes an entry for
shared resources that don't belong to a single database. Handle
this in tests and add docs and example document.
Handle metrics renamed in `pg_stat_statements` in version 13.
Regenerate data files with latest code and PostgreSQL 13.

(cherry picked from commit be4885e)

CHANGELOG.next.asciidoc

@@ -670,6 +670,7 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for Darwin/arm M1. {pull}24019[24019]
 - Check fields are documented in aws metricsets. {pull}23887[23887]
 - Add support for defining metrics_filters for prometheus module in hints. {pull}24264[24264]
+- Add support for PostgreSQL 10, 11, 12 and 13. {pull}24402[24402]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -38995,6 +38995,14 @@ TCP port number that the client is using for communication with this backend, or
 
 type: long
 
+--
+
+*`postgresql.activity.backend_type`*::
++
+--
+Type of the current backend. Possible types are autovacuum launcher, autovacuum worker, logical replication launcher, logical replication worker, parallel worker, background writer, client backend, checkpointer, startup, walreceiver, walsender and walwriter. Extensions may register workers with additional backend types.
+
+
 --
 
 *`postgresql.activity.backend_start`*::
@@ -39071,6 +39079,22 @@ Current overall state of this backend. Possible values are:
 Text of this backend's most recent query. If state is active this field shows the currently executing query. In all other states, it shows the last query that was executed.
 
 
+--
+
+*`postgresql.activity.wait_event`*::
++
+--
+Wait event name if the backend is currently waiting.
+
+
+--
+
+*`postgresql.activity.wait_event_type`*::
++
+--
+The type of event for which the backend is waiting.
+
+
 --
 
 [float]
@@ -39200,7 +39224,7 @@ One row per database, showing database-wide statistics. Collected by querying pg
 *`postgresql.database.oid`*::
 +
 --
-OID of the database this backend is connected to.
+OID of the database this backend is connected to, or 0 for shared resources.
 
 
 type: long
@@ -39210,7 +39234,7 @@ type: long
 *`postgresql.database.name`*::
 +
 --
-Name of the database this backend is connected to.
+Name of the database this backend is connected to, empty for shared resources.
 
 
 type: keyword

metricbeat/docs/modules/postgresql.asciidoc

@@ -61,7 +61,7 @@ precedence over those specified in the `username` and `password` config options.
 [float]
 === Compatibility
 
-This module was tested with PostgreSQL 9.5.3 and is expected to work with all
+This module was tested with PostgreSQL 9, 10, 11, 12 and 13. It is expected to work with all
 versions >= 9.
 
 

metricbeat/mb/testing/fetcher.go

@@ -96,6 +96,8 @@ func (f *reportingMetricSetV2FetcherError) WriteEvents(t testing.TB, path string
 }
 
 func (f *reportingMetricSetV2FetcherError) WriteEventsCond(t testing.TB, path string, cond func(common.MapStr) bool) {
+ t.Helper()
+
  err := WriteEventsReporterV2ErrorCond(f, t, path, cond)
  if err != nil {
  t.Fatal("writing events", err)

metricbeat/module/postgresql/_meta/docs.asciidoc

@@ -54,5 +54,5 @@ precedence over those specified in the `username` and `password` config options.
 [float]
 === Compatibility
 
-This module was tested with PostgreSQL 9.5.3 and is expected to work with all
+This module was tested with PostgreSQL 9, 10, 11, 12 and 13. It is expected to work with all
 versions >= 9.

metricbeat/module/postgresql/_meta/supported-versions.yml

@@ -0,0 +1,6 @@
+variants:
+ - POSTGRESQL_VERSION: "9.6.21"
+ - POSTGRESQL_VERSION: "10.16"
+ - POSTGRESQL_VERSION: "11.11"
+ - POSTGRESQL_VERSION: "12.6"
+ - POSTGRESQL_VERSION: "13.2"

metricbeat/module/postgresql/activity/_meta/data.json

@@ -1,45 +1,41 @@
 {
  "@timestamp": "2017-10-12T08:05:34.853Z",
- "agent": {
- "hostname": "host.example.com",
- "name": "host.example.com"
- },
  "event": {
  "dataset": "postgresql.activity",
  "duration": 115000,
  "module": "postgresql"
  },
  "metricset": {
- "name": "activity"
+ "name": "activity",
+ "period": 10000
  },
  "postgresql": {
  "activity": {
  "application_name": "",
- "backend_start": "2019-03-05T08:38:21.348Z",
+ "backend_start": "2021-03-05T19:01:40.467Z",
+ "backend_type": "client backend",
  "client": {
- "address": "172.26.0.1",
+ "address": "192.168.128.1",
  "hostname": "",
- "port": 41582
+ "port": 33414
  },
  "database": {
  "name": "postgres",
- "oid": 12379
+ "oid": 13395
  },
- "pid": 347,
+ "pid": 2236,
  "query": "SELECT * FROM pg_stat_activity",
- "query_start": "2019-03-05T08:38:21.352Z",
- "state": "active",
- "state_change": "2019-03-05T08:38:21.352Z",
- "transaction_start": "2019-03-05T08:38:21.352Z",
+ "query_start": "2021-03-05T19:01:40.469Z",
+ "state": "idle",
+ "state_change": "2021-03-05T19:01:40.471Z",
  "user": {
  "id": 10,
  "name": "postgres"
- },
- "waiting": false
+ }
  }
  },
  "service": {
- "address": "172.26.0.2:5432",
+ "address": "192.168.128.2:5432",
  "type": "postgresql"
  }
 }

metricbeat/module/postgresql/activity/_meta/data_backend.json

@@ -0,0 +1,25 @@
+{
+ "@timestamp": "2017-10-12T08:05:34.853Z",
+ "event": {
+ "dataset": "postgresql.activity",
+ "duration": 115000,
+ "module": "postgresql"
+ },
+ "metricset": {
+ "name": "activity",
+ "period": 10000
+ },
+ "postgresql": {
+ "activity": {
+ "backend_start": "2021-03-05T18:39:18.347Z",
+ "backend_type": "logical replication launcher",
+ "pid": 81,
+ "wait_event": "LogicalLauncherMain",
+ "wait_event_type": "Activity"
+ }
+ },
+ "service": {
+ "address": "192.168.128.2:5432",
+ "type": "postgresql"
+ }
+}

metricbeat/module/postgresql/activity/_meta/fields.yml

@@ -39,6 +39,12 @@
  description: >
  TCP port number that the client is using for communication with this
  backend, or -1 if a Unix socket is used.
+ - name: backend_type
+ description: >
+ Type of the current backend. Possible types are autovacuum launcher, autovacuum worker,
+ logical replication launcher, logical replication worker, parallel worker, background
+ writer, client backend, checkpointer, startup, walreceiver, walsender and walwriter.
+ Extensions may register workers with additional backend types.
  - name: backend_start
  type: date
  description: >
@@ -79,4 +85,9 @@
  Text of this backend's most recent query. If state is active this field
  shows the currently executing query. In all other states, it shows the
  last query that was executed.
-
+ - name: wait_event
+ description: >
+ Wait event name if the backend is currently waiting.
+ - name: wait_event_type
+ description: >
+ The type of event for which the backend is waiting.

metricbeat/module/postgresql/activity/activity.go

@@ -22,6 +22,7 @@ import (
 
  "github.com/pkg/errors"
 
+ "github.com/elastic/beats/v7/libbeat/common"
  "github.com/elastic/beats/v7/metricbeat/mb"
  "github.com/elastic/beats/v7/metricbeat/module/postgresql"
 )
@@ -56,13 +57,22 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 // of an error set the Error field of mb.Event or simply call report.Error().
 func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
  ctx := context.Background()
+
  results, err := m.QueryStats(ctx, "SELECT * FROM pg_stat_activity")
  if err != nil {
  return errors.Wrap(err, "error in QueryStats")
  }
 
  for _, result := range results {
- data, _ := schema.Apply(result)
+ var data common.MapStr
+ // If the activity is not connected to any database, it is from a backend service. This
+ // can be distingished by checking if the record has a database identifier (`datid`).
+ // Activity records on these cases have different sets of fields.
+ if datid, ok := result["datid"].(string); ok && datid != "" {
+ data, _ = schema.Apply(result)
+ } else {
+ data, _ = backendSchema.Apply(result)
+ }
  reporter.Event(mb.Event{
  MetricSetFields: data,
  })

metricbeat/module/postgresql/activity/activity_integration_test.go

@@ -22,12 +22,12 @@ package activity
 import (
  "testing"
 
+ "github.com/stretchr/testify/assert"
+
  "github.com/elastic/beats/v7/libbeat/common"
  "github.com/elastic/beats/v7/libbeat/tests/compose"
  mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
  "github.com/elastic/beats/v7/metricbeat/module/postgresql"
-
- "github.com/stretchr/testify/assert"
 )
 
 func TestFetch(t *testing.T) {
@@ -43,26 +43,38 @@ func TestFetch(t *testing.T) {
 
  t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
- // Check event fields
- assert.Contains(t, event, "database")
- db_oid := event["database"].(common.MapStr)["oid"].(int64)
- assert.True(t, db_oid > 0)
-
  assert.Contains(t, event, "pid")
  assert.True(t, event["pid"].(int64) > 0)
 
- assert.Contains(t, event, "user")
- assert.Contains(t, event["user"].(common.MapStr), "name")
- assert.Contains(t, event["user"].(common.MapStr), "id")
+ // Check event fields
+ if _, isQuery := event["database"]; isQuery {
+ db_oid := event["database"].(common.MapStr)["oid"].(int64)
+ assert.True(t, db_oid > 0)
+
+ assert.Contains(t, event, "user")
+ assert.Contains(t, event["user"].(common.MapStr), "name")
+ assert.Contains(t, event["user"].(common.MapStr), "id")
+ } else {
+ assert.Contains(t, event, "backend_type")
+ assert.Contains(t, event, "wait_event")
+ assert.Contains(t, event, "wait_event_type")
+ }
 }
 
 func TestData(t *testing.T) {
  service := compose.EnsureUp(t, "postgresql")
 
- f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host()))
- if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil {
- t.Fatal("write", err)
- }
+ f := mbtest.NewFetcher(t, getConfig(service.Host()))
+
+ dbNameKey := "postgresql.activity.database.name"
+ f.WriteEventsCond(t, "", func(event common.MapStr) bool {
+ _, err := event.GetValue(dbNameKey)
+ return err == nil
+ })
+ f.WriteEventsCond(t, "./_meta/data_backend.json", func(event common.MapStr) bool {
+ _, err := event.GetValue(dbNameKey)
+ return err != nil
+ })
 }
 
 func getConfig(host string) map[string]interface{} {

metricbeat/module/postgresql/activity/data.go

@@ -48,4 +48,14 @@ var schema = s.Schema{
  "waiting": c.Bool("waiting", s.Optional),
  "state": c.Str("state"),
  "query": c.Str("query"),
+ "backend_type": c.Str("backend_type", s.Optional),
+}
+
+// Fields available in events from backend activity.
+var backendSchema = s.Schema{
+ "pid": c.Int("pid"),
+ "backend_start": c.Time(time.RFC3339Nano, "backend_start"),
+ "wait_event_type": c.Str("wait_event_type"),
+ "wait_event": c.Str("wait_event"),
+ "backend_type": c.Str("backend_type"),
 }

metricbeat/module/postgresql/bgwriter/_meta/data.json

@@ -1,44 +1,41 @@
 {
  "@timestamp": "2017-10-12T08:05:34.853Z",
- "agent": {
- "hostname": "host.example.com",
- "name": "host.example.com"
- },
  "event": {
  "dataset": "postgresql.bgwriter",
  "duration": 115000,
  "module": "postgresql"
  },
  "metricset": {
- "name": "bgwriter"
+ "name": "bgwriter",
+ "period": 10000
  },
  "postgresql": {
  "bgwriter": {
  "buffers": {
- "allocated": 143,
+ "allocated": 273,
  "backend": 0,
  "backend_fsync": 0,
- "checkpoints": 0,
+ "checkpoints": 8,
  "clean": 0,
  "clean_full": 0
  },
  "checkpoints": {
  "requested": 0,
- "scheduled": 1,
+ "scheduled": 4,
  "times": {
  "sync": {
- "ms": 0
+ "ms": 9
  },
  "write": {
- "ms": 0
+ "ms": 804
  }
  }
  },
- "stats_reset": "2019-03-05T08:32:30.028Z"
+ "stats_reset": "2021-03-05T18:39:17.954Z"
  }
  },
  "service": {
- "address": "172.26.0.2:5432",
+ "address": "192.168.128.2:5432",
  "type": "postgresql"
  }
 }