diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f8f7a1f510c..e8d7dae10cc 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -670,6 +670,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for Darwin/arm M1. {pull}24019[24019] - Check fields are documented in aws metricsets. {pull}23887[23887] - Add support for defining metrics_filters for prometheus module in hints. {pull}24264[24264] +- Add support for PostgreSQL 10, 11, 12 and 13. {pull}24402[24402] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d01808d94c8..0ec5e7695c0 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -38995,6 +38995,14 @@ TCP port number that the client is using for communication with this backend, or type: long +-- + +*`postgresql.activity.backend_type`*:: ++ +-- +Type of the current backend. Possible types are autovacuum launcher, autovacuum worker, logical replication launcher, logical replication worker, parallel worker, background writer, client backend, checkpointer, startup, walreceiver, walsender and walwriter. Extensions may register workers with additional backend types. + + -- *`postgresql.activity.backend_start`*:: @@ -39071,6 +39079,22 @@ Current overall state of this backend. Possible values are: Text of this backend's most recent query. If state is active this field shows the currently executing query. In all other states, it shows the last query that was executed. +-- + +*`postgresql.activity.wait_event`*:: ++ +-- +Wait event name if the backend is currently waiting. + + +-- + +*`postgresql.activity.wait_event_type`*:: ++ +-- +The type of event for which the backend is waiting. + + -- [float] @@ -39200,7 +39224,7 @@ One row per database, showing database-wide statistics. Collected by querying pg *`postgresql.database.oid`*:: + -- -OID of the database this backend is connected to. +OID of the database this backend is connected to, or 0 for shared resources. type: long @@ -39210,7 +39234,7 @@ type: long *`postgresql.database.name`*:: + -- -Name of the database this backend is connected to. +Name of the database this backend is connected to, empty for shared resources. type: keyword diff --git a/metricbeat/docs/modules/postgresql.asciidoc b/metricbeat/docs/modules/postgresql.asciidoc index 1958a3b314e..ef475199d6a 100644 --- a/metricbeat/docs/modules/postgresql.asciidoc +++ b/metricbeat/docs/modules/postgresql.asciidoc @@ -61,7 +61,7 @@ precedence over those specified in the `username` and `password` config options. [float] === Compatibility -This module was tested with PostgreSQL 9.5.3 and is expected to work with all +This module was tested with PostgreSQL 9, 10, 11, 12 and 13. It is expected to work with all versions >= 9. diff --git a/metricbeat/mb/testing/fetcher.go b/metricbeat/mb/testing/fetcher.go index e80101c0918..aa2a427d80e 100644 --- a/metricbeat/mb/testing/fetcher.go +++ b/metricbeat/mb/testing/fetcher.go @@ -96,6 +96,8 @@ func (f *reportingMetricSetV2FetcherError) WriteEvents(t testing.TB, path string } func (f *reportingMetricSetV2FetcherError) WriteEventsCond(t testing.TB, path string, cond func(common.MapStr) bool) { + t.Helper() + err := WriteEventsReporterV2ErrorCond(f, t, path, cond) if err != nil { t.Fatal("writing events", err) diff --git a/metricbeat/module/postgresql/_meta/docs.asciidoc b/metricbeat/module/postgresql/_meta/docs.asciidoc index 8d2dceb4850..f4baa6f7e88 100644 --- a/metricbeat/module/postgresql/_meta/docs.asciidoc +++ b/metricbeat/module/postgresql/_meta/docs.asciidoc @@ -54,5 +54,5 @@ precedence over those specified in the `username` and `password` config options. [float] === Compatibility -This module was tested with PostgreSQL 9.5.3 and is expected to work with all +This module was tested with PostgreSQL 9, 10, 11, 12 and 13. It is expected to work with all versions >= 9. diff --git a/metricbeat/module/postgresql/_meta/supported-versions.yml b/metricbeat/module/postgresql/_meta/supported-versions.yml new file mode 100644 index 00000000000..99cc23ede98 --- /dev/null +++ b/metricbeat/module/postgresql/_meta/supported-versions.yml @@ -0,0 +1,6 @@ +variants: + - POSTGRESQL_VERSION: "9.6.21" + - POSTGRESQL_VERSION: "10.16" + - POSTGRESQL_VERSION: "11.11" + - POSTGRESQL_VERSION: "12.6" + - POSTGRESQL_VERSION: "13.2" diff --git a/metricbeat/module/postgresql/activity/_meta/data.json b/metricbeat/module/postgresql/activity/_meta/data.json index a8b9f7ac513..7b3c51fa056 100644 --- a/metricbeat/module/postgresql/activity/_meta/data.json +++ b/metricbeat/module/postgresql/activity/_meta/data.json @@ -1,45 +1,41 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "dataset": "postgresql.activity", "duration": 115000, "module": "postgresql" }, "metricset": { - "name": "activity" + "name": "activity", + "period": 10000 }, "postgresql": { "activity": { "application_name": "", - "backend_start": "2019-03-05T08:38:21.348Z", + "backend_start": "2021-03-05T19:01:40.467Z", + "backend_type": "client backend", "client": { - "address": "172.26.0.1", + "address": "192.168.128.1", "hostname": "", - "port": 41582 + "port": 33414 }, "database": { "name": "postgres", - "oid": 12379 + "oid": 13395 }, - "pid": 347, + "pid": 2236, "query": "SELECT * FROM pg_stat_activity", - "query_start": "2019-03-05T08:38:21.352Z", - "state": "active", - "state_change": "2019-03-05T08:38:21.352Z", - "transaction_start": "2019-03-05T08:38:21.352Z", + "query_start": "2021-03-05T19:01:40.469Z", + "state": "idle", + "state_change": "2021-03-05T19:01:40.471Z", "user": { "id": 10, "name": "postgres" - }, - "waiting": false + } } }, "service": { - "address": "172.26.0.2:5432", + "address": "192.168.128.2:5432", "type": "postgresql" } } \ No newline at end of file diff --git a/metricbeat/module/postgresql/activity/_meta/data_backend.json b/metricbeat/module/postgresql/activity/_meta/data_backend.json new file mode 100644 index 00000000000..ffbaee2acc0 --- /dev/null +++ b/metricbeat/module/postgresql/activity/_meta/data_backend.json @@ -0,0 +1,25 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "postgresql.activity", + "duration": 115000, + "module": "postgresql" + }, + "metricset": { + "name": "activity", + "period": 10000 + }, + "postgresql": { + "activity": { + "backend_start": "2021-03-05T18:39:18.347Z", + "backend_type": "logical replication launcher", + "pid": 81, + "wait_event": "LogicalLauncherMain", + "wait_event_type": "Activity" + } + }, + "service": { + "address": "192.168.128.2:5432", + "type": "postgresql" + } +} \ No newline at end of file diff --git a/metricbeat/module/postgresql/activity/_meta/fields.yml b/metricbeat/module/postgresql/activity/_meta/fields.yml index 6a5ea2a63aa..02576a5021a 100644 --- a/metricbeat/module/postgresql/activity/_meta/fields.yml +++ b/metricbeat/module/postgresql/activity/_meta/fields.yml @@ -39,6 +39,12 @@ description: > TCP port number that the client is using for communication with this backend, or -1 if a Unix socket is used. + - name: backend_type + description: > + Type of the current backend. Possible types are autovacuum launcher, autovacuum worker, + logical replication launcher, logical replication worker, parallel worker, background + writer, client backend, checkpointer, startup, walreceiver, walsender and walwriter. + Extensions may register workers with additional backend types. - name: backend_start type: date description: > @@ -79,4 +85,9 @@ Text of this backend's most recent query. If state is active this field shows the currently executing query. In all other states, it shows the last query that was executed. - + - name: wait_event + description: > + Wait event name if the backend is currently waiting. + - name: wait_event_type + description: > + The type of event for which the backend is waiting. diff --git a/metricbeat/module/postgresql/activity/activity.go b/metricbeat/module/postgresql/activity/activity.go index 38c8f0ec621..38558f18071 100644 --- a/metricbeat/module/postgresql/activity/activity.go +++ b/metricbeat/module/postgresql/activity/activity.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/postgresql" ) @@ -56,13 +57,22 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { ctx := context.Background() + results, err := m.QueryStats(ctx, "SELECT * FROM pg_stat_activity") if err != nil { return errors.Wrap(err, "error in QueryStats") } for _, result := range results { - data, _ := schema.Apply(result) + var data common.MapStr + // If the activity is not connected to any database, it is from a backend service. This + // can be distingished by checking if the record has a database identifier (`datid`). + // Activity records on these cases have different sets of fields. + if datid, ok := result["datid"].(string); ok && datid != "" { + data, _ = schema.Apply(result) + } else { + data, _ = backendSchema.Apply(result) + } reporter.Event(mb.Event{ MetricSetFields: data, }) diff --git a/metricbeat/module/postgresql/activity/activity_integration_test.go b/metricbeat/module/postgresql/activity/activity_integration_test.go index 47a91c18ad4..5cf46489d7f 100644 --- a/metricbeat/module/postgresql/activity/activity_integration_test.go +++ b/metricbeat/module/postgresql/activity/activity_integration_test.go @@ -22,12 +22,12 @@ package activity import ( "testing" + "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/tests/compose" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/metricbeat/module/postgresql" - - "github.com/stretchr/testify/assert" ) func TestFetch(t *testing.T) { @@ -43,26 +43,38 @@ func TestFetch(t *testing.T) { t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) - // Check event fields - assert.Contains(t, event, "database") - db_oid := event["database"].(common.MapStr)["oid"].(int64) - assert.True(t, db_oid > 0) - assert.Contains(t, event, "pid") assert.True(t, event["pid"].(int64) > 0) - assert.Contains(t, event, "user") - assert.Contains(t, event["user"].(common.MapStr), "name") - assert.Contains(t, event["user"].(common.MapStr), "id") + // Check event fields + if _, isQuery := event["database"]; isQuery { + db_oid := event["database"].(common.MapStr)["oid"].(int64) + assert.True(t, db_oid > 0) + + assert.Contains(t, event, "user") + assert.Contains(t, event["user"].(common.MapStr), "name") + assert.Contains(t, event["user"].(common.MapStr), "id") + } else { + assert.Contains(t, event, "backend_type") + assert.Contains(t, event, "wait_event") + assert.Contains(t, event, "wait_event_type") + } } func TestData(t *testing.T) { service := compose.EnsureUp(t, "postgresql") - f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) - if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { - t.Fatal("write", err) - } + f := mbtest.NewFetcher(t, getConfig(service.Host())) + + dbNameKey := "postgresql.activity.database.name" + f.WriteEventsCond(t, "", func(event common.MapStr) bool { + _, err := event.GetValue(dbNameKey) + return err == nil + }) + f.WriteEventsCond(t, "./_meta/data_backend.json", func(event common.MapStr) bool { + _, err := event.GetValue(dbNameKey) + return err != nil + }) } func getConfig(host string) map[string]interface{} { diff --git a/metricbeat/module/postgresql/activity/data.go b/metricbeat/module/postgresql/activity/data.go index 7307c4467bf..6275917e5cc 100644 --- a/metricbeat/module/postgresql/activity/data.go +++ b/metricbeat/module/postgresql/activity/data.go @@ -48,4 +48,14 @@ var schema = s.Schema{ "waiting": c.Bool("waiting", s.Optional), "state": c.Str("state"), "query": c.Str("query"), + "backend_type": c.Str("backend_type", s.Optional), +} + +// Fields available in events from backend activity. +var backendSchema = s.Schema{ + "pid": c.Int("pid"), + "backend_start": c.Time(time.RFC3339Nano, "backend_start"), + "wait_event_type": c.Str("wait_event_type"), + "wait_event": c.Str("wait_event"), + "backend_type": c.Str("backend_type"), } diff --git a/metricbeat/module/postgresql/bgwriter/_meta/data.json b/metricbeat/module/postgresql/bgwriter/_meta/data.json index af0511c2a4c..1d21e7a6edf 100644 --- a/metricbeat/module/postgresql/bgwriter/_meta/data.json +++ b/metricbeat/module/postgresql/bgwriter/_meta/data.json @@ -1,44 +1,41 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "dataset": "postgresql.bgwriter", "duration": 115000, "module": "postgresql" }, "metricset": { - "name": "bgwriter" + "name": "bgwriter", + "period": 10000 }, "postgresql": { "bgwriter": { "buffers": { - "allocated": 143, + "allocated": 273, "backend": 0, "backend_fsync": 0, - "checkpoints": 0, + "checkpoints": 8, "clean": 0, "clean_full": 0 }, "checkpoints": { "requested": 0, - "scheduled": 1, + "scheduled": 4, "times": { "sync": { - "ms": 0 + "ms": 9 }, "write": { - "ms": 0 + "ms": 804 } } }, - "stats_reset": "2019-03-05T08:32:30.028Z" + "stats_reset": "2021-03-05T18:39:17.954Z" } }, "service": { - "address": "172.26.0.2:5432", + "address": "192.168.128.2:5432", "type": "postgresql" } } \ No newline at end of file diff --git a/metricbeat/module/postgresql/database/_meta/data.json b/metricbeat/module/postgresql/database/_meta/data.json index 1c6b7b66c99..d3eef4ac6ba 100644 --- a/metricbeat/module/postgresql/database/_meta/data.json +++ b/metricbeat/module/postgresql/database/_meta/data.json @@ -1,20 +1,19 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" + "event": { + "dataset": "postgresql.database", + "duration": 115000, + "module": "postgresql" }, "metricset": { - "host": "postgresql:5432", - "module": "postgresql", "name": "database", - "rtt": 115 + "period": 10000 }, "postgresql": { "database": { "blocks": { - "hit": 0, - "read": 0, + "hit": 43969, + "read": 381, "time": { "read": { "ms": 0 @@ -26,24 +25,29 @@ }, "conflicts": 0, "deadlocks": 0, - "name": "template1", - "number_of_backends": 0, - "oid": 1, + "name": "postgres", + "number_of_backends": 2, + "oid": 13395, "rows": { - "deleted": 0, - "fetched": 0, - "inserted": 0, - "returned": 0, - "updated": 0 + "deleted": 42, + "fetched": 26418, + "inserted": 98, + "returned": 53201, + "updated": 14 }, + "stats_reset": "2021-03-05T18:39:18.089Z", "temporary": { "bytes": 0, "files": 0 }, "transactions": { - "commit": 0, - "rollback": 0 + "commit": 445, + "rollback": 5 } } + }, + "service": { + "address": "192.168.128.2:5432", + "type": "postgresql" } } \ No newline at end of file diff --git a/metricbeat/module/postgresql/database/_meta/data_shared.json b/metricbeat/module/postgresql/database/_meta/data_shared.json new file mode 100644 index 00000000000..6437f4e74d3 --- /dev/null +++ b/metricbeat/module/postgresql/database/_meta/data_shared.json @@ -0,0 +1,53 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "postgresql.database", + "duration": 115000, + "module": "postgresql" + }, + "metricset": { + "name": "database", + "period": 10000 + }, + "postgresql": { + "database": { + "blocks": { + "hit": 5314, + "read": 31, + "time": { + "read": { + "ms": 0 + }, + "write": { + "ms": 0 + } + } + }, + "conflicts": 0, + "deadlocks": 0, + "name": "", + "number_of_backends": 0, + "oid": 0, + "rows": { + "deleted": 0, + "fetched": 1212, + "inserted": 0, + "returned": 2033, + "updated": 0 + }, + "stats_reset": "2021-03-05T18:39:18.089Z", + "temporary": { + "bytes": 0, + "files": 0 + }, + "transactions": { + "commit": 0, + "rollback": 0 + } + } + }, + "service": { + "address": "192.168.128.2:5432", + "type": "postgresql" + } +} \ No newline at end of file diff --git a/metricbeat/module/postgresql/database/_meta/fields.yml b/metricbeat/module/postgresql/database/_meta/fields.yml index c39a3a6d0f3..2b08d1630de 100644 --- a/metricbeat/module/postgresql/database/_meta/fields.yml +++ b/metricbeat/module/postgresql/database/_meta/fields.yml @@ -8,11 +8,11 @@ - name: oid type: long description: > - OID of the database this backend is connected to. + OID of the database this backend is connected to, or 0 for shared resources. - name: name type: keyword description: > - Name of the database this backend is connected to. + Name of the database this backend is connected to, empty for shared resources. - name: number_of_backends type: long description: > diff --git a/metricbeat/module/postgresql/database/database_integration_test.go b/metricbeat/module/postgresql/database/database_integration_test.go index b64e410e73a..08d17b23576 100644 --- a/metricbeat/module/postgresql/database/database_integration_test.go +++ b/metricbeat/module/postgresql/database/database_integration_test.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/module/postgresql" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestFetch(t *testing.T) { @@ -45,7 +46,7 @@ func TestFetch(t *testing.T) { // Check event fields db_oid := event["oid"].(int64) - assert.True(t, db_oid > 0) + assert.True(t, db_oid >= 0) assert.Contains(t, event, "name") _, ok := event["name"].(string) assert.True(t, ok) @@ -61,10 +62,28 @@ func TestFetch(t *testing.T) { func TestData(t *testing.T) { service := compose.EnsureUp(t, "postgresql") - f := mbtest.NewReportingMetricSetV2Error(t, getConfig(service.Host())) - if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { - t.Fatal("write", err) + getOid := func(event common.MapStr) int { + oid, err := event.GetValue("postgresql.database.oid") + require.NoError(t, err) + + switch oid := oid.(type) { + case int: + return oid + case int64: + return int(oid) + } + t.Log(event) + t.Fatalf("no numeric oid in event: %v (%T)", oid, oid) + return 0 } + + f := mbtest.NewFetcher(t, getConfig(service.Host())) + f.WriteEventsCond(t, "", func(event common.MapStr) bool { + return getOid(event) != 0 + }) + f.WriteEventsCond(t, "./_meta/data_shared.json", func(event common.MapStr) bool { + return getOid(event) == 0 + }) } func getConfig(host string) map[string]interface{} { diff --git a/metricbeat/module/postgresql/docker-compose.yml b/metricbeat/module/postgresql/docker-compose.yml index 25d813835c2..d1367aefe66 100644 --- a/metricbeat/module/postgresql/docker-compose.yml +++ b/metricbeat/module/postgresql/docker-compose.yml @@ -2,10 +2,12 @@ version: '2.3' services: postgresql: - image: docker.elastic.co/integrations-ci/beats-postgresql:${POSTGRESQL_VERSION:-9.5.3}-1 + image: docker.elastic.co/integrations-ci/beats-postgresql:${POSTGRESQL_VERSION:-13.2}-2 build: context: ./_meta args: - POSTGRESQL_VERSION: ${POSTGRESQL_VERSION:-9.5.3} + POSTGRESQL_VERSION: ${POSTGRESQL_VERSION:-13.2} + environment: + POSTGRES_PASSWORD: postgres ports: - 5432 diff --git a/metricbeat/module/postgresql/fields.go b/metricbeat/module/postgresql/fields.go index 75826021f79..4856aebc115 100644 --- a/metricbeat/module/postgresql/fields.go +++ b/metricbeat/module/postgresql/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPostgresql returns asset data. // This is the base64 encoded gzipped contents of module/postgresql. func AssetPostgresql() string { - return "eJzUWkuP2zgSvvevKMwlycIRsNc+LLDIHDbAJpNBsmejTJUswhSpkJTd3l+/KFIvS/Kzpd6MT922+NXHqmK9qI+wo+MzlMb5rSX3Uz0BeOkVPcNv3+KX3//8929PACk5YWXppdHP8I8nAIAv5K0UDoRRioSnFDJrCujWgSO7J+uSJwCXG+vXwuhMbp8hQ+XoCcCSInT0DFt8AsgkqdQ9B/CPoLGgATX++GPJz1tTlfU3E9T40+NRRKZJ/VtfTl8WCi/30h/bH6akXZDInz80QWpEVZD2UJKtdQClNYKcW7EiDlJvQerM2AIZg9WArD9vwOcEorKWtD/BbbiBycDn6HuAlcgBHTiPngB12qyHnxXZYwKfWvtsjieY4XfmUm7XvHrdCEl6j52aqPkMVdhXY4oeN+goMTI9eaBRpzJ6O/jhgkaDVj//HjdOLTr4XDrYoNiRTkGyG2odt+lNcpkY/zvJbEfHg7FD1lfIfcWCZmBXzqatb9E1oFFax2RacuXIJkvYioFBme2WUpA6ePdNXCbsc4cNHpCKZamkCIdx/TrhPaR4Tge2v4GMUJK0TzBNLTl3H5XP36Be1xCKaA9yyI3z9+vjX8b5gNNyaIVH3BXHK0ulsTEqAYIlzhQEv3/9DsqYXVXy4vj4mrd0kScjzeS+Pz59A4YDXRUbstGIPUVKB5XjoJkZC8IURaUbex+kz4NuR6C1rldgLHz8O8gMEP6j5Qs4I3ZUg9IZW9SLOUSf2WWK/k4T/ZAFwSEnHX2hTiZwiHmEzbICmVCyah6adKQRLD8XE970VrxF7TjLGP0G23nXZsKe3P4ep0mGtLgYvTa9q2NM6hQFnureWHaSmNKlA22GVOqKgHoGUuj8GGt6jwF5LXLU2+lM+NpNRurMI9CKks6QOaD0cnROI4+NMYpQ30nFVsT6GyXhVvO1SDAaEJQRuwtquk/2p9rlzJ4sKlUrYpiHuTp1cqMI9qgqcoCWumpqBArwt9rez/Ajp/6e6IVEFfaCdcE3uVqmary20QKHMgRNh+6QFwUOc0MfCqTuH6oRsmS99h5Ywaby5zyZP51p7tjQgAW8x01IKR+Yj3Td+XGykAot57563SSJE8L0Iqj0YHSbxgIcF/Zhf/xNX7hADuGTuKiBrDV2eicZOl+izyGrdAOl1EVD85KPJ2umoVPpcKMoHeqjzb18SCyKXVP6S3L8e7Mu7vNasRCsdOcJpRc/PBTvHBRcOVgSve7lcy8M1vEyLAodyAiXuys3iLKd4hpIDXwyjc+5P2NwtwLpu8Uj2F5oDfUAx7UI249po4Zysz1Y6cm+pqH87tFL57nRxo2pYi3CGmMMnUKU0KY813aK/cYvVizDnTVtX0NzfNzub/5ETmJXGqm9S5zIKa3U6FA8Wpp9jRWZyaBF7suLpslxT7Ah0tx9c4t9Luv0mVr6WZHzCzBtkWdi6mVBLgn2SophoRnpZsrgMMheO47GowIsTKXjseRcXpN0kaMrQ/Ub4x5HDw5UXKK35EaotU+y6x1ysgSZVDHVBa/1XCoYjjW7FQMXUinpSBidulsV4Y5a/JX1wPxza7T8b+zM7lDGpsoysi7pKWV2761ltOZKK8u76NvhMreJ2m1+VpvjdFC8gds6q5SanWDwzTOB2nlTlpQCQiDA6nQCNWwoFBAgx/6TY9o7MAYK1Md2Gxf3WGfW5f1CWhKca0Mvf7FYGFBbZ3wEFrJASyWo0JsmZ4P0DsxBQxAeyi14r40tUKlhIQNn7JijTlWwsnEEtCfdNT+N1NRwORVljWADzoeLOkKljMAl0lJjwFbC+f7HrS05mrUV5goqlyJnZblYV9clzoHDY6i2gtBkVFI1A9bXlFR/aAJrDmE83+B1g/nmm48Hmfa5nQ7S2+H5ZEU1wfL+UupXG5//klPzOKVbm2xdL1sgCdbAvZZiPE1t7xauzb1cws31KMrPEPJ6Mtq+rafZk4ozcjh77k/4WqMU6+D/y5hZ8MnDc7OajTJix6U8zh8uuSyrBQALGLG9SClfwtohwfV5hdCZhRSFikkemyo1BnsQKHKOc1NjW/RhBs+FBoYhDWjimhXtEd6HnRqtGFCoKiUHueyGIN1F6wj4VDLD8gJTksXQkLuj81S8c6Eirv+LT3+4qFHefbD0udr/gbk/J6ZY3DNwkwkis1rFm2MXDIYesJoaKt1Qxfc2dLGre9WOGPmtdmTNgU+hr6xeop82Bz6CEb3Jw2FqddOJDOQy8iJfilsN/iA1qR3ZReYQzK1Bf5BcVaaLFKOBWw3+ILWUFC1GrQa/n5owOlNSLNCYNzwEakGcF9OKuBZpJcbLSEvC7MkeG74jxCtlCxWlsWiPSZhYzJ/EGvx6IiIsXfUB+OeoYYcREFoCYSod7tIsbdFyrxbuxA95nBacLuG0N0Jt6LynZJtw4rTxzoibP5dLvf2wCu/bnAoIV3Fmu2YB6ym9ATjyjHRN6Zujn03pw8lWyAS9KdxQg2MTnPcdNskDJrhixVCJ1CZYQs8pYRqS4PzVYoMMKfnYJtwWL36ddnvUd7d3X69tvE9ejotXKvxXeGlnuiGfeFPuBFXqvanfwGhejgu43atxoqygcriNr8f5cBRCxXXHu3Hd5d/rLkje8mWrbhIVtGIxFuwTt6pv/u5e/60HW2kY3mCe3C3Op6+OS2vPS3I9vdx5g/BnvCQcrjuFFajUAtm0nTpH1bZXFra6qFyucWbNMvpytX7G9WqN87EMJ3T2a52OVr+JaXq+SuvuivgawULqGel9kVoWVTErQXyZkyC+zE6Q8KwK7/e7L4R6TnbOpynt5+P3zZSViinKedQp2hRS2ssua/WmD32mN94FRuoFFcYeE5ejpXTGqdPw+EQBcYQQpzVxHlTfw11V8SnPGQd2NxANM67HiKbSejlbj3kD11rgg3Tr4v7t6A6uY2+lq4xAtaC3BvxXO2tkuaCvjmk+4qqR5rKeOmb6oKNGssv66Zjsg27KbeaS9mf8V5s/kFxWoSOe0/r8XwAAAP//xMZFkg==" + return "eJzUWk+P47oNv8+nIN7l7RbZoL3OoUCxr0AX6O7bh92ix4CR6ViILHklOZn00xeU5D+xncwkY8+2Oc0kFvkTKZE/kv4Aezo9QmWc31lyP9QDgJde0SP88jV++e2Pf/7yAJCRE1ZWXhr9CH99AAD4TN5K4UAYpUh4yiC3poRuHTiyB7Ju/QDgCmP9Rhidy90j5KgcPQBYUoSOHmGHDwC5JJW5xyD8A2gsaQCNP/5U8fPW1FX6ZgIaf3o4yoh0nX7r6+nrQuHlQfpT+8OUtisa+fO7JsiMqEvSHiqyyQZQWSPIuRUb4ij1DqTOjS2RZbAZkO3nDfiCQNTWkvZnchtsYHLwBfqewFoUgA6cR0+AOmvWw4+a7GkNH1v/bE9nMsPvjKXabXj1plGy7j127qLmMzRh34wZetyio7WR2dkDjTmV0bvBD1csGqz66be4cWqlgy+kgy2KPekMJB9DreM2vVlfB8b/TiLb0+lo7BD1M+C+YEkzoKtms9bXeDSgMVqHZFpz7ciul/AVCwZldjvKQOpwul+EZcI/N/jgDq1YVUqKcBk3r1PekxTv6cD3LwAjlCTt15hllpy7Dcqnr5DWNYCitDsxFMb52+3xD+N8kNNiaJVHuSuOV5YqY2NUAgRLnCkIfvvyDZQx+7rixfHxDW/pKk6WNNPx/f7xK7A40HW5JRud2DOkdFA7Dpq5sSBMWda68fdR+iLYdiQ02XoFxsKHv4DMAeFfWj6BM2JPSShd8EVavOEd3biXU9X5ICWFxu2cH53cKgqWcoCWAGtvDijqugSFtRYF2VX/y6Oxe7KrkR5ldlKgYpe2h78TMPVrkgQVWlSKVPsFw+N0q4fhCOBopedHkiNam4qCxL4yUodfnUfr62oFR1SWBMkDf3tkwqEzsiFBHlFFYeuRkr8/edJOGu2gxBNY2knnySZ8LvoYs0zyNlC1IT4Y8br/ArLJU5qhv9WzsiQ4FqTjXU5kAI6RB/C1WoFc03rVPDQZCEZi+blIWKa34i1qxyzB6DfYzq/toe3p7e9xGmSgNYvBa2+SOkVSRlHhue2N5UseKZl0oM0QSmJ01HOQQufHsqb3GCRvRIF6N81kXrvJCJ1xBFhR0wUwR5RejuJsxLE1RhHqG6HYmth+IxLVWj6pBKMBQRmxv2Km23R/TEfOHIhDUzLEkEd10fOAqo7hs2PDI6EAf0r+foTvBfX3RE8k6rAXTIR9crXM1HhtYwVORQiajt0lL0sc5va+KJC6f6lGkiXbtffACra1v3SS+dO55oYNDVDAO9wGSvCe8UjX3R8nS6nQMndJ6yZBnAGmJ0GVB6PbFBjEcWEW9sff9JUL5BQ8KRc1kLVmIl3wTnJ0vkJfQF7rRpRSVx3NSz6crZkWnUmHW0XZ0B4td+JLYlHsm9JNkuPfm3Vxn8+RveClG28oPfnhpfjVQcnMj7NuV31+6oXBFC/DolBBjuRydewGUbYzXCNSA99M4wuur1m4W4H03eIxQelCa+BzHNei2GsxbUOHYRn+rGH+jdJDWBcZsIxH71oQew7APcSviLSOXRTBcIg4FlIUQzgjEC1t2UWO9JpuyDePXjovhQPcmtq3yiPFS5SuzfeubXP0uxaRbg/d2vQsGpjjWHN756Jjkm7tREFZrUYR4d664kssJ0wOreS+vnguCzwQbIk0VGRzY8tLx7OP1NKPmpxfAGkreSakXpbk1sFf63JYJUW4uTJ445X7bjwqwNLUOsYkJjIJpIsYXRVKtxj0OXRylOb6sgU3kprOJB+9Y0GWIJcqlUm8Ac88yXCg3a9YcCmVko6E0dmFMmBsCHfS4v/ZDoy/sEbL/8S2wg3G2NZ5Ttate0aZ/fQmHa27stryLvp+uI5tgrjOj2p7mg6KL8C2yWulZgcYzuaFQO28qSrKACEAYHM6gRq2FNgTyPH5KTDrXRgDJepTu42re0xJavlzIS0JzsehEXWVKQ2gbXK+Agt5oIUSTOhNQ1hAegfmqCEoD1wT3mljS1RqyOLggh8L1JkKXjaOAkPoKr9Ga2aYS0Zd072Y91dthEoZgUukpcaBrYbLxZ/bWHI0ax+A6WNDpFwsKhLFOXJ4DFQzKB1TqmY68BpK9bsmsOYYZkuNvG6q1Hzz4SizPrbzKVA7+ZlkVBMob6dSP2/2E5ovfw6E1xVoKWNvmNqKS/25nzwNWgGVlT/dAjjchI3JN0nkArkzCe5VK+MJQjtPe65X6NbClOUoOcwQKXs62lq3Z/UzohoxXAwXZ3itUYpt8HMRMwq+sHipv7VVRuy5AsD5oyyzuaQAWMEI7VVIxRLeDnmxjytE3DxkNlQM8tSQ25gjQKAoODxOtbrRh7kT8xMMjS3QxFQX7QnehZ0arVigUHVGDgrZNY66lwtGgs81s1heYCqyGJoY7uQ8lb+6QKTTf/Hp91ctyrsPnr5UMtwx6+J8FmsCFtwkkIgsmXh76oLB8ASsphpxLyD/vQ1dLQZftSOW/FY7subIt9DXVi9RhpsjX8EovUnfodP3ohsZwOXkRbEUtiT8TmhSO7KLtC8YWyP9TnB1lS3CYQO2JPxOaBkpWgxaEn47NGF0rqRYoJ5vcAjUgjgvZjUxF2k1xuGsJWEOZE8N3pHEZ2gLlZWxaE/r0OiYP4k18lMjRVh69gzA30Z1PowEoSUQptZh/mhph5ZLvPAeyLGITYbzJZz2RlIbOO9ovVtz4rRxzsY1oyuk3r1fhRH6uYIwvjS7DSvYTNkNwJG/3OzujL49+dmMPmyIhUzQa94NLTh2weWzwy65wwXPeDEwkeSCJeycEWYhCc7PFhvJkJGPZcLL4sX/TpU+KtfbeeFr6/WzF0LjGIr/Ci+qTdfxE2+HnkmV+mDSqzTNC6FBbvc6qKhqqB3u4iuhPlyFwLhueB+0G5i+bq7yli8Ydg2sYBWLkbBPTKLf/H3V/psittYwnPqezWPns1eHpfXnNb2enm4cPPwRB6vDdediBSq1QDZtm9XRtO2kw9ZXjcscZ9Yso6+z9QtHL1mcr2W4obNPgzpY/SKmqflqrbux+nMAS6lnhPdZalnW5awA8WlOgPg0O0DCiya8/dx9JtRzonM+y+gwH76vpqpVTFHOo87QZpDRQXZZq9d96CN94QgxQi+pNPa0jo3TGbtOw+uTOrOhhRC7NbEflMZ3z5r4HOeMDbsXAA09rvuAZtJ6OVuN+QKsSeGdcBO5fzu4gynuS+EqI1AteFqD/Fcf1ohywbM6hnnPUY0wlz2pY6R3HtQIdtlzOgZ75zHlMnNJ/7P8V7s/gFzWoCOc0/b8bwAAAP//1pQyXQ==" } diff --git a/metricbeat/module/postgresql/statement/_meta/data.json b/metricbeat/module/postgresql/statement/_meta/data.json index 6b5cc15a4b9..c6ff97bd5f2 100644 --- a/metricbeat/module/postgresql/statement/_meta/data.json +++ b/metricbeat/module/postgresql/statement/_meta/data.json @@ -12,11 +12,11 @@ "postgresql": { "statement": { "database": { - "oid": 12379 + "oid": 13395 }, "query": { - "calls": 2, - "id": 1592910677, + "calls": 132, + "id": -3489238739385425370, "memory": { "local": { "dirtied": 0, @@ -26,7 +26,7 @@ }, "shared": { "dirtied": 0, - "hit": 0, + "hit": 924, "read": 0, "written": 0 }, @@ -35,23 +35,23 @@ "written": 0 } }, - "rows": 3, - "text": "SELECT * FROM pg_stat_statements", + "rows": 396, + "text": "SELECT d.datname as \"Name\",\n pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n d.datcollate as \"Collate\",\n d.datctype as \"Ctype\",\n pg_catalog.array_to_string(d.datacl, $1) AS \"Access privileges\"\nFROM pg_catalog.pg_database d\nORDER BY 1", "time": { "max": { - "ms": 0.161 + "ms": 0.325369 }, "mean": { - "ms": 0.1595 + "ms": 0.07867374242424244 }, "min": { - "ms": 0.158 + "ms": 0.053835 }, "stddev": { - "ms": 0.0015000000000000013 + "ms": 0.037920252272212004 }, "total": { - "ms": 0.319 + "ms": 10.384934000000003 } } }, @@ -61,7 +61,7 @@ } }, "service": { - "address": "192.168.144.2:5432", + "address": "192.168.128.2:5432", "type": "postgresql" } } \ No newline at end of file diff --git a/metricbeat/module/postgresql/statement/data.go b/metricbeat/module/postgresql/statement/data.go index 9829a75b1d5..9206de59a3a 100644 --- a/metricbeat/module/postgresql/statement/data.go +++ b/metricbeat/module/postgresql/statement/data.go @@ -22,7 +22,7 @@ import ( c "github.com/elastic/beats/v7/libbeat/common/schema/mapstrstr" ) -// Based on: https://www.postgresql.org/docs/9.6/pgstatstatements.html +// Based on: https://www.postgresql.org/docs/13/pgstatstatements.html var schema = s.Schema{ "user": s.Object{ "id": c.Int("userid"), @@ -36,11 +36,11 @@ var schema = s.Schema{ "calls": c.Int("calls"), "rows": c.Int("rows"), "time": s.Object{ - "total": s.Object{"ms": c.Float("total_time")}, - "min": s.Object{"ms": c.Float("min_time")}, - "max": s.Object{"ms": c.Float("max_time")}, - "mean": s.Object{"ms": c.Float("mean_time")}, - "stddev": s.Object{"ms": c.Float("stddev_time")}, + "total": s.Object{"ms": c.Float("total_exec_time")}, + "min": s.Object{"ms": c.Float("min_exec_time")}, + "max": s.Object{"ms": c.Float("max_exec_time")}, + "mean": s.Object{"ms": c.Float("mean_exec_time")}, + "stddev": s.Object{"ms": c.Float("stddev_exec_time")}, }, "memory": s.Object{ "shared": s.Object{ @@ -62,3 +62,18 @@ var schema = s.Schema{ }, }, } + +// PostgreSQL 13 renames fields with stats about execution time from *_time to *_exec_time, +// keep this for compatibility with older versions. +// Based on: https://www.postgresql.org/docs/9.6/pgstatstatements.html +var schemaOldTime = s.Schema{ + "query": s.Object{ + "time": s.Object{ + "total": s.Object{"ms": c.Float("total_time")}, + "min": s.Object{"ms": c.Float("min_time")}, + "max": s.Object{"ms": c.Float("max_time")}, + "mean": s.Object{"ms": c.Float("mean_time")}, + "stddev": s.Object{"ms": c.Float("stddev_time")}, + }, + }, +} diff --git a/metricbeat/module/postgresql/statement/statement.go b/metricbeat/module/postgresql/statement/statement.go index c874ec81ada..d2ef253e8ad 100644 --- a/metricbeat/module/postgresql/statement/statement.go +++ b/metricbeat/module/postgresql/statement/statement.go @@ -66,6 +66,12 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { for _, result := range results { data, _ := schema.Apply(result) + // Older versions of PostgreSQL had the execution time in `*_time` fields + // instead of `*_exec_time`, keep compatibility with them. + if _, ok := result["total_time"]; ok { + execTimes, _ := schemaOldTime.Apply(result) + data.DeepUpdate(execTimes) + } reporter.Event(mb.Event{ MetricSetFields: data, }) diff --git a/metricbeat/module/postgresql/test_postgresql.py b/metricbeat/module/postgresql/test_postgresql.py index 6ac3c0aa131..675dda44667 100644 --- a/metricbeat/module/postgresql/test_postgresql.py +++ b/metricbeat/module/postgresql/test_postgresql.py @@ -5,6 +5,7 @@ import unittest +@metricbeat.parameterized_with_supported_versions class Test(metricbeat.BaseTest): COMPOSE_SERVICES = ['postgresql'] @@ -21,13 +22,10 @@ def common_checks(self, output): def get_hosts(self): username = "postgres" + password = os.getenv("POSTGRESQL_PASSWORD", default="postgres") host = self.compose_host() dsn = "postgres://{}?sslmode=disable".format(host) - return ( - [dsn], - username, - os.getenv("POSTGRESQL_PASSWORD"), - ) + return ([dsn], username, password) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') @@ -52,9 +50,12 @@ def test_activity(self): self.common_checks(output) for evt in output: - assert "name" in evt["postgresql"]["activity"]["database"] - assert "oid" in evt["postgresql"]["activity"]["database"] - assert "state" in evt["postgresql"]["activity"] + if "database" in evt["postgresql"]["activity"]: + assert "name" in evt["postgresql"]["activity"]["database"] + assert "oid" in evt["postgresql"]["activity"]["database"] + assert "state" in evt["postgresql"]["activity"] + else: + assert "backend_type" in evt["postgresql"]["activity"] @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") @pytest.mark.tag('integration') @@ -112,3 +113,33 @@ def test_bgwriter(self): assert "checkpoints" in evt["postgresql"]["bgwriter"] assert "buffers" in evt["postgresql"]["bgwriter"] assert "stats_reset" in evt["postgresql"]["bgwriter"] + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + @pytest.mark.tag('integration') + def test_statement(self): + """ + PostgreSQL module outputs an event. + """ + hosts, username, password = self.get_hosts() + self.render_config_template(modules=[{ + "name": "postgresql", + "metricsets": ["statement"], + "hosts": hosts, + "username": username, + "password": password, + "period": "5s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + + output = self.read_output_json() + self.common_checks(output) + + for evt in output: + statement = evt["postgresql"]["statement"] + assert "user" in statement + assert "database" in statement + assert "query" in statement + assert "ms" in statement["query"]["time"]["max"] + assert "ms" in statement["query"]["time"]["total"]