From f42e2feb9ae4138b21da382807fb82c5f305808c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Sat, 11 Apr 2020 21:49:19 -0600 Subject: [PATCH] [Metricbeat] add more metric collection into pubsub metricset (#17600) (#17651) * add more metric collection into pubsub metricset * fix docs.asciidoc for pubsub (cherry picked from commit 50c6e869a7b970cd0e5f326dc1c505b6b1ff6007) --- CHANGELOG.next.asciidoc | 1 + .../metricbeat/module/googlecloud/fields.go | 2 +- .../module/googlecloud/pubsub/_meta/data.json | 28 +++++- .../pubsub/_meta/data_subscription.json | 53 ----------- .../googlecloud/pubsub/_meta/data_topic.json | 61 ------------- .../googlecloud/pubsub/_meta/docs.asciidoc | 40 ++++++++- .../googlecloud/pubsub/_meta/fields.yml | 88 +++++++++++++++++++ .../module/googlecloud/pubsub/manifest.yml | 28 ++++++ 8 files changed, 180 insertions(+), 121 deletions(-) delete mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json delete mode 100644 x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d688f822eb3..29dba4ff142 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -195,6 +195,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix vsphere VM dashboard host aggregation visualizations. {pull}17555[17555] - Fix azure storage dashboards. {pull}17590[17590] - Metricbeat no longer needs to be started strictly after Logstash for `logstash-xpack` module to report correct data. {issue}17261[17261] {pull}17497[17497] +- Fix pubsub metricset to collect all GA stage metrics from gcp stackdriver. {issue}17154[17154] {pull}17600[17600] - Add privileged option so as mb to access data dir in Openshift. {pull}17606[17606] - Fix "ID" event generator of Google Cloud module {issue}17160[17160] {pull}17608[17608] - Fix storage metricset to allow config without region/zone. {issue}17623[17623] {pull}17624[17624] diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index 3f8bc6b15c1..8fac4947787 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzsXE1z2zgSvedX9C3JlqMc9ubamipHszuTWmfHtXZyVYFgS8QaBLj4sEb59VMASIqURBKUKE08pZstkujXDeC9RgPkB3jGzS2spFxxpFza9A2AYYbjLbz9xf8Kc/czPHBillLlb98AKORINN5Cgoa8AUhRU8UKw6S4hZ/eAAD8Mn+AXKaW4xuAJUOe6lt/4QMIkmPL5IyTBLn2lwHMpsBbkMn/kJryp+bzzTasRjX7W/3zwWe3d+doSEoMGfWEYlTHP6A32mAefz+VeWENNm7fDW27mZWStmj82gp8q7vmoeXKhcYzu8Fs4mFCGyIoti52Ge9qrNngkilcE873buhrtK/hZuOpkkWB6SLZGNQLKq0wB++vbHEpVh03tAL5WVCZM7EC33BlBpINmAz7XNqHVhD6jOYs4Mqmo+HVY66wZ+kNhRrVC6YLKhXqXl9TaRO+O8gOevsfmyeoQC7Bt1obASm8t5nUxl11f3cM3jZKaxhn34lrfSKITy7oilD3XwWFcC4pMZjC/OErmIwYYBqoVQqF4RtgwnFX5UQccE1WuDAs7wI1FvdX1yAspXJoy/AyARqpFOl+/9Ujm+nnM40fMv1UnruWXK+EqexswFLJvMuNHTiyOAsYZzxg+fwbyAKVH4+H50yFZ62YwXPHxxkxKMDI4QAFQOeMkLcwEKJ6ehQdU6MXQMv4r3Lt7/Nz8tsXyIiGBFGAskIwsbqJmR4CzVqqc80QiuzlDIK3N0uCnTBTXDS6vTqAbnrNq/FVmnccQo3CnDt2zgbIF1TjUF0gZv3IKjRckjQhnAjKWnamykvvJUnhU2VgZHqaGVPsTr3uaXUCiC4gTTCJi6tIF5wYFJQdTHlaEO4cqxnFEttMFcLjG6CEU8t9zuAH9TrDkB0o/L9FbWBNyj4sE71Cyd83jqndPyUYsMIw3ri8P1GqO4Ntbfy4dWAU6kIKjbOzcNcJA3u7GiP9OdtgC4l1UyFMsv4MNbYpWXQr91DYYCB0TXO/9dqJsxVjr2nzn78XUqAwjPBP3ts+8/EQYmE0oayUXJtssSTUSDX4VFQuvG9EUxJxdwQP7zctbL5YMuEzuKhQDtjZ5aCSIwbELT4VcmsaUS+9GspGdMVHOlDKr09PDx8fvWxA0A0nMLJCpvfZpAv71KhrnOWKMdnUoNzlQ8BjwAaSvFCkg7Ey1DX6d1IBJTTD9y7S4xxZKimM88TQYqHMIfARovXfpydHx9oqJy1SARKaAZVCYFgHJ2jWLnWmnHlnRBoE6SouV3G5istg4z+EuDBhUAkyZe12TO7818mehwdv/5CNobmIkRNBdSNaiaG78c31U14cCwzP/3jii+edONY5mv7GEGA8BR5PgiNpcDwRjqLCE8lwwNa2whWTZQ8CPzLTDqmUjk/59oGfA/KBNHs8vKikesrAthLrrjVMGfE+6EYawi8sY7UqtaRsV7tW0sDd/N/NsQNSBNGq/PdhuSrWSc1dFetkIPFg4KpYEbbG6dU5qkKjtSpWqSYsBo0F9ucVfkbrU7w6vUJtupaLOpq6lotGQ4iFAddy0UE79a7x32cd1aILb89WgHClUOtesu4JXRxRe/b6fP9pWxSqKBneLaWCp/kDLLlca2DmrQ6ssz1VKAWQouCM+oMtoI1CkoMUfPN+l/V2nOo/K3CcW61DAj2OlZrg3OpCycSlYl9iMrIF9jyxr7y6TPAPetYFTRnTo/EDCt/ayfGnQ1zotjs52m/vOBQ+nPuC3Ee0w4XgI9f6gzIc9fSwBI9ppk9+p1lNDwnvtEvNkZJ7nlXnOLEdIbXjhDZaZk8Q2QGJNbRYaM0XPoP+c0W2MX251P7Ed00XJ62Jmqe9twTkj0+vUSEYVDkTfglSUdXHx8f7rq3lEcnAcSh3Benbl4ZUWu1P6fdDm2Y7XudSmgxTz+bvmIBcv9+yenMp9lZ7fteG0OebsGufM2ENtgSRkw2q0ruC6HIZWm+ZBQeva7Pr2uy6Nhts/AfZyr8UBW4z129f4ihQ4PpiAkIVjlYPWaCYDOA8vALUSP6lNdoQkbpItUEraVeZp+4OpBXCwibaJg3TUx3WfrDJo01GHtLWNqnbOy5PebT1P84V32PH5SeEPi9y1P7VqdNH/dzmlhPDXjDIlOu+snXtTAm55piuguTebf+vq883wfdwQ4qcvaDaeAD9Jw+5XIWpe1rFWRrCw4EVzb5jlTxY0YJeO/SOzJ5nZFYBqC+8ByaAtLq5Z3LbfNEY4VVnTDXLa7BlMNHXKtrg3moorM4ARVpIJswNJNaAkAY2aFrd1u+GFbWRqd04axdInqI2C29jC31BVqe9LXW3Qp9qli9Eva+GUzDX4VKXR6McKizni+bMrl8PO9Mcb3hSO7J9Ja01qRVqy80M/iUVEEjRi3mZqR96VGODaz9uybYVio+5TL3DKZKUM4FdnkfEbLoNtqFIHSS9Mj79OC/Vr87Yq+7Q83amD8+Rvaizs4PTGRBjMC8Og4OvgrNn9F7om7ACds/4DWAFLC845ihMWPGmErXXhIQYmvkPS9RsO4NHGZbK1U6nFHzjkjVDmNAgBbYemPnybtOYckMg1J1RKancOHEytWIvKFrPAiW+No1EQW65YQVHMCzHnl3eVrT7NnvjI/7zTpHBO1N5X5sIpQZGlaxEYNQo8S8+Xig7q47+tqemkdW0dEJcbqcfk6SFHQSX4FxcnB4r0w9uuv4FpOpALM9LJbXBNuPBmpkMhBQfHMdsWqFl6ThC3PHpUmNjx7NLDYh/UJniT0cNi9gI1sdxLjQkyvM4g52+PXlTMHrc6vfJPXriuveVRAvaapBeMPNLONPZq6XIad+b7ItQVP73obPi5L8u5ncbUuWE9IRiVPwXrbazQKr2MneqwthjaHlkZYwULJoS+o+39Pd97CmEn5Ebsu3yu4fPQInPlht97RJndyVHk8nU2/evc9bn9BzZHyZBYk32/USPm7U8vkiIxnQRPqa2IJSinqKavROHOgHwdeMw1n19QpSfcYMEHdmtFBGOogMO0JIj30Bq0eWV5Z138/uezN25tOWPCTwJ33RyfTm/bzATJEo+o1vlrIXv095ktgyvLpCyJaMLhzK3Zio+3ok2zYhwGXpO0mbgKusHItj/qZ3jB1jk53WO9nTn4zq7n2I5ddbBuC/dnOjGwa/JTEUc+8R9Ws+WI3qCQ9u+hN7YvPENayhQlTu9rRCUfgDlROsZPGVMwwvhFoHpxqkrQdG3kJLNjQ+RC119n8IibFoRE6oQmuQFdwTkj1y+EF7tFUlr/KMp6dnOCmfCXScuur5l9TrG+NaRKbZFqh0Rwnndqyycb//BOvaPAAAA//+LJduy" + return "eJzsXUuT2zYSvvtX9M321nh82JtrK1WTyW7iWnsztTPJVQWCLQk7IMDgMYry67cAkBQp8QFSpPwo3WYkEv11A/i+bgCk3sEz7j/ARsoNR8qlTV8BGGY4foDXP/tP4d59DA+cmLVU2etXAAo5Eo0fIEFDXgGkqKliuWFSfIAfXgEA/Hz/AJlMLcdXAGuGPNUf/BfvQJAMGyZvOUmQa/81gNnn+AFk8j+kpviofn+9DatR3f6t+rj13sPVGRqSEkNG3aEY1fE36L02mMVfT2WWW4O1y49D22xmo6TNa582At/orvvQculC7Z7jYNbxMKENERQbX3YZ72qs3uCaKdwRzk8u6Gu0r+F646mSeY7pKtkb1CsqrTCt15e2uBSbjgsagfwoqMyY2IBvuDQDyR7MFvtcOoWWE/qMZhFwRdPR8Koxl9tFekOhRvWC6YpKhbrX11TahB8PslZv/2OzBBXINfhWKyMghfd2K7Vx37q/OwZvE6U1jLO/iGt9JohPLuiKUPdfCYVwLikxmML9w29gtsQA00CtUigM3wMTjrtKJ+KAa7LBlWFZF6ixuH9zDcJaKoe2CC8ToJFKkZ72XzWymX5eaPyQ+afyvWvJ9UqYys4GrJXMutw4giPzRcA44wHLx19B5qj8eGyfMyWenWIGl46PM2JQgJHDAQqAloyQtzAQomp65B1ToxdAw/gvcuev83Py98+wJRoSRAHKCsHE5iZmegg0O6mWmiEU2csCgncyS4KdMFNcNLq9akE3v+ZV+ErNm4ZQozBLx87ZAPmCahyqC8SsH1mJhkuSJoQTQVnDzlx56SdJUvixNDAyPd0akx9Pve5pdQaILiB1MImLq0hXnBgUlLWmPA0Id47VjGKJracK4fY9UMKp5T5n8IN6t8WQHSj8w6I2sCNFHxaJXq7kn3vH1O6fAgxYYRivfX06Ucorg21t/Lh1YBTqXAqNt4tw1xkD+1CNkf6cbbCFxLqpECZZf4Ya25TMu5V7KGwwELq6uV977cTZirFXt/nPP3MpUBhG+I/e2z7z8RBiYdShbJTcme1qTaiRavCuqFz41IimJOLqCB4+bVrYbLVmwmdwUaEcsHPMQQVHDIhbfCrkahpRlV41ZSO65CMdKOWXp6eH949eNiDohhMYWSLTp2zShX1u1BXOomJM9hUo93Ub8BiwgSQvFOlgrAh1hf6NVEAJ3eJbF+lxjqyVFMZ5Ymi+UqYNfIRo/ffpydGxtspJi1SAhG6BSiEw1MEJmp1LnSln3hmRBkG6istVXK7iMtj4VyEuTBhUgsy5djsmd/5+sufhwds/ZGNoLmLkRFDdiFZi6G58c/2UF8cCw/M/nvjieSeOdSbT3xgCjKfA6SQ4kgbHE+EoKjyTDAdsHVa4YrLsQeATM+2QSun4lO8U+BKQW9Ls8fCikuo5A9tIrLtqmCLifdCNNIRfWMYqVWpI2bF2baSBu/t/18cOSBFEq/Tfh+WqWGc1d1Wss4HEg4GrYkXYGqdXS6wKjdaqWKWacTFoLLAvt/AzWp/i1ekb1KbrclFHU9flotEQYmHAdbmo1U61a/z3247Vogtvz5aAcKNQ616y7gldHFF79vr46cfDolBJyfBmLRU83T/AmsudBmZe68A6h1OFUgDJc86oP9gC2igkGUjB92+PWe/Iqf6zAtPcahwS6HGs0ATnVhdKJi4V+wKTkQ2wy8S+9OoywW/1rAuaMqZH4wcUvrGT40+HuNAddnK0395xKHw4TwW5j2iHF4In1vqDMhx197AEj2mmT37nqaaHhHfeUnOk5C5TdY4T2xFSO05oo2X2DJEdkFhD85XWfOUz6C8rsrXpy6X2J74rujirJqqf9j4QkD8+vUOFYFBlTPgSpKSq94+Pn7q2lkckA9NQHgvS759rUmm1P6XfD22e7XidSWm2mHo2f8MEZPrtgdXrpdhr7fldG0Kfb8KufcaENdgQRE72qArvcqKLMrTaMgsOXmuza212rc0GG/9KtvIvRYGHzPX3z3EUKHB3MQGhCkerh8xRzAbwPjwCVEv+pTXaEJG6SDVBK2k3W0/dHUhLhLlNtE1qpuc6rP1gk0ebjDykrW1StTctT3m01T/OFd9j0/ITQp9XGWr/6NT5o/7eZpYTw14wyJTrvqJ17UwJueOYboLk3h3+r1afb4Lv4YIUOXtBtfcA+k8ecrkJU/e8FWdpCA8HVjT7C8vkwYoG9MqhN+T2+ZbclgCqL94CE0Aa3dwzuW22qo3wsjPmmuUV2CKY6NcqmuBea8it3gKKNJdMmBtIrAEhDezRNLqt3w0rKiNzu7FoF0ieojYrb+MAfUU25z0tdbdBn2oWD0S9LYdTMNfhUpdHoxzKLeer+syuHg9baI7XPKkcOTyS1pjUCrXl5hb+JRUQSNGLeZGpt92qsca17w9k2wjF+0ym3uEUScqZwC7PI2I23wbbUKRaSa+ITz/OS/WrM/ZNd+iynenDM7EX9XZxcHoLxBjM8nZw8Jvg7Bm9F/omVMDuHr8BrIBlOccMhQkVbypRe01IiKFb/2KJim1v4VGGUrnc6ZSC712yZggTGqTAxg23fnm3bky5IRDWnVEpqdw4cTK1YS8oGvcCJX5tGomCzHLDco5gWIY9u7yNaPdt9sZH/KejRQbvTOl9ZSIsNTCqZCkCo0aJf/DxQtlZefS3OTWNLKelE+JiO31KkhZ2EFyCc3FxeixNP7jp+h1IVUssl6WSymCT8WDHzBaEFO8cx+wboWXpOEI88ulSY+PIs0sNiH9QmeIPk4ZFbASr4zgXGhLFeZxRne58XXE0BtWlaC63CWd6G8ogZx+CfTAyZ3QU+s4eW9iF3VZqhNKqP1Rk89SvASR7+CxTtt7f0eefygvOqa0nj8pzvT11I25uxnuyLGVGdsMQ6qI0VegSKfQeXK5Cba1PSygTCuxuL1bJfqVw0/7KoS/lzw0kSj6jS3134pAaBZxTVhMu42XHusJc7h114mKLbu3rPVNGXxvimbpiJuwTu0IjLp75IT73FpHRCd2XEpJm9n9KyxPlZMi9ryIhP/U2bAeM8tSN7hWVekYvwhvp6pGvtuKZCBtmt/C0ZRqYdsW/T9aKS+APKw2pv6WuGzqVYs02q5AdzbGr19YXwYgNrgDdEuEme/XIf3Oe1+Jeue/t+xcADHVFqSkXoLCuBfdzuOtw6D5ndNrG15O79cwtr2+wUNIo0gsu+voS6ZtdHZn3lSl9EVompf9qkuGiJP7usuBz/Po6k8lzPJpJ388T9Zqce+n0It+zov/llP170fCxQ6aiV0FyvZXH0Y49uFLcfaaEF+xyoXWQjqWCwpVBDrw49/XjnUoTS5+y6aqTB8PcQLbsjBrAODG0wmaLHfsZG8w6lpliGY1qYvi+HS1413nu0P/GhD9znir2gvWTuWOPJMb/rsGhIJKqSaFzHY98DC2PPB9JchatLv0POfaXAbHPov2E3JDDULh7+AiU+DMTte539OC+ydBsZertl2MgPK1NZYrtkkqs2f51psf1E518lRCN6Sr8pMaKUIp6jnlxFIdq1cmfHg5j3U9oUfyYByTo6t6NIsJJfcABWnLke0gtugSwuPLu/lNPtudcOiSWM3gS3uzv+vL+Uy1lPeaf/r2xIrw6R8rWjK4cysyauUrzo2iXC0sZSeuBK623RPBwfLztFdvTB1jkS9Yne3r0ivXjF3KfO+vg+IDNom60vlN8LuI4Je7zerYY0TO8usOnT7Uj/L5hDTmq4nmfRggKP4Byossi8YVwi65UPDx7Kyj6FlKyv/EhcqGrrlOYh0cXiClWXkmWc0dA/sH7F8LLJwakNf7WlPQ81BDeDOKL465fNPg2xvjBkTnS9jIbJpxXvcrCW06+so79fwAAAP//qGqEJg==" } diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json index 42516375e59..86da8d924e5 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data.json @@ -14,14 +14,34 @@ "googlecloud": { "labels": { "resource": { - "topic_id": "test-ks" + "subscription_id": "test-ks" } }, "pubsub": { + "snapshot": { + "backlog_bytes": 19, + "backlog_bytes_by_region": 19, + "num_messages": 4, + "num_messages_by_region": 4, + "oldest_message_age": 69319, + "oldest_message_age_by_region": 69319 + }, "subscription": { - "backlog_bytes": 20, - "num_undelivered_messages": 4, - "oldest_unacked_message_age": 435752 + "backlog_bytes": 0, + "num_undelivered_messages": 0, + "oldest_retained_acked_message_age": 0, + "oldest_retained_acked_message_age_by_region": 0, + "oldest_unacked_message_age": 0, + "oldest_unacked_message_age_by_region": 69277, + "retained_acked_bytes": 0, + "retained_acked_bytes_by_region": 0, + "unacked_bytes_by_region": 19 + }, + "topic": { + "oldest_retained_acked_message_age_by_region": 0, + "oldest_unacked_message_age_by_region": 69319, + "retained_acked_bytes_by_region": 0, + "unacked_bytes_by_region": 76 } } }, diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json deleted file mode 100644 index 1503d6aa80f..00000000000 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "@timestamp": "2020-01-14T11:46:00.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "googlecloud": { - "labels": { - "resource": { - "subscription_id": "test-subscription" - } - }, - "pubsub": { - "subscription": { - "num_undelivered_messages": 0, - "oldest_unacked_message_age": 0, - "backlog_bytes": 0 - } - } - }, - "service": { - "type": "googlecloud" - }, - "cloud": { - "account": { - "id": "elastic-metricbeat" - }, - "provider": "googlecloud" - }, - "host": { - "name": "mcastro" - }, - "agent": { - "version": "8.0.0", - "type": "metricbeat", - "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa", - "hostname": "mcastro", - "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228" - }, - "ecs": { - "version": "1.4.0" - }, - "event": { - "dataset": "googlecloud.pubsub", - "module": "googlecloud", - "duration": 933264720 - }, - "metricset": { - "name": "pubsub", - "period": 300000 - } -} diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json deleted file mode 100644 index f85d8148782..00000000000 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "@timestamp": "2020-01-14T11:42:00.000Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "ecs": { - "version": "1.4.0" - }, - "cloud": { - "provider": "googlecloud", - "account": { - "id": "elastic-metricbeat" - } - }, - "metricset": { - "name": "pubsub", - "period": 300000 - }, - "event": { - "duration": 932060852, - "dataset": "googlecloud.pubsub", - "module": "googlecloud" - }, - "googlecloud": { - "labels": { - "resource": { - "subscription_id": "test-subscription" - } - }, - "pubsub": { - "topic": { - "message_sizes": { - "bucket_options": { - "Options": { - "ExponentialBuckets": { - "growth_factor": 4, - "scale": 1, - "num_finite_buckets": 16 - } - } - } - } - } - } - }, - "service": { - "type": "googlecloud" - }, - "host": { - "name": "mcastro" - }, - "agent": { - "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa", - "hostname": "mcastro", - "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", - "version": "8.0.0", - "type": "metricbeat" - } -} diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc index 9fa3ea4f50c..cd6e94083ef 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc @@ -1,25 +1,61 @@ PubSub Metricset to fetch metrics from https://cloud.google.com/pubsub/[Pub/Sub] topics and subscriptions in Google Cloud Platform. -The `pubsub` Metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub[Stackdriver API]. The field names have been left untouched for people already familiar with them. +The `pubsub` Metricset contains all GA stage metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub[Stackdriver API]. The field names have been left untouched for people already familiar with them. No special permissions are needed apart from the ones detailed in the module section of the docs. +[float] +==== Snapshot Metrics +- `pubsub.snapshot.backlog_bytes`: Total byte size of the messages retained in a snapshot. +- `pubsub.snapshot.backlog_bytes_by_region`: Total byte size of the messages retained in a snapshot, broken down by Cloud region. +- `pubsub.snapshot.config_updates_count`: Cumulative count of configuration changes, grouped by operation type and result. +- `pubsub.snapshot.num_messages`: Number of messages retained in a snapshot. Sampled every 60 seconds. +- `pubsub.snapshot.num_messages_by_region`: Number of messages retained in a snapshot, broken down by Cloud region. +- `pubsub.snapshot.oldest_message_age`: Age (in seconds) of the oldest message retained in a snapshot. +- `pubsub.snapshot.oldest_message_age_by_region`: Age (in seconds) of the oldest message retained in a snapshot, broken down by Cloud region. + +[float] +==== Subscription Metrics - `pubsub.subscription.ack_message_count`: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. - `pubsub.subscription.backlog_bytes`: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.byte_cost`: Cumulative cost of operations, measured in bytes. This is used to measure quota utilization. +- `pubsub.subscription.config_updates_count`: Cumulative count of configuration changes for each subscription, grouped by operation type and result. +- `pubsub.subscription.dead_letter_message_count`: Cumulative count of messages published to dead letter topic, grouped by result. +- `pubsub.subscription.mod_ack_deadline_message_count`: Cumulative count of messages whose deadline was updated by ModifyAckDeadline requests, grouped by delivery type. +- `pubsub.subscription.mod_ack_deadline_message_operation_count`: Cumulative count of ModifyAckDeadline message operations, grouped by result. +- `pubsub.subscription.mod_ack_deadline_request_count`: Cumulative count of ModifyAckDeadline requests, grouped by result. - `pubsub.subscription.num_outstanding_messages`: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. - `pubsub.subscription.num_undelivered_messages`: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. +- `pubsub.subscription.oldest_retained_acked_message_age`: Age (in seconds) of the oldest acknowledged message retained in a subscription. +- `pubsub.subscription.oldest_retained_acked_message_age_by_region`: Age (in seconds) of the oldest acknowledged message retained in a subscription, broken down by Cloud region. - `pubsub.subscription.oldest_unacked_message_age`: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. +- `pubsub.subscription.oldest_unacked_message_age_by_region`: Age (in seconds) of the oldest unacknowledged message in a subscription, broken down by Cloud region. - `pubsub.subscription.pull_ack_message_operation_count`: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - `pubsub.subscription.pull_ack_request_count`: Cumulative count of acknowledge requests, grouped by result. - `pubsub.subscription.pull_message_operation_count`: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - `pubsub.subscription.pull_request_count`: Cumulative count of pull requests, grouped by result. - `pubsub.subscription.push_request_count`: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. - `pubsub.subscription.push_request_latencies`: Distribution of push request latencies (in microseconds), grouped by result. +- `pubsub.subscription.retained_acked_bytes`: otal byte size of the acknowledged messages retained in a subscription. +- `pubsub.subscription.retained_acked_bytes_by_region`: Total byte size of the acknowledged messages retained in a subscription, broken down by Cloud region. +- `pubsub.subscription.seek_request_count`: Cumulative count of seek attempts, grouped by result. - `pubsub.subscription.sent_message_count`: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. - `pubsub.subscription.streaming_pull_ack_message_operation_count`: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - `pubsub.subscription.streaming_pull_ack_request_count`: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. - `pubsub.subscription.streaming_pull_message_operation_count`: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count +- `pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation_count`: Cumulative count of StreamingPull ModifyAckDeadline operations, grouped by result. +- `pubsub.subscription.streaming_pull_mod_ack_deadline_request_count`: Cumulative count of streaming pull requests with non-empty ModifyAckDeadline fields, grouped by result. - `pubsub.subscription.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. -- `pubsub.topic.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result. +- `pubsub.subscription.unacked_bytes_by_region`: Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. + +[float] +==== Topic Metrics +- `pubsub.topic.byte_cost`: Cost of operations, measured in bytes. This is used to measure utilization for quotas. +- `pubsub.topic.config_updates_count`: Cumulative count of configuration changes, grouped by operation type and result. +- `pubsub.topic.message_sizes`: Distribution of publish message sizes (in bytes). +- `pubsub.topic.oldest_retained_acked_message_age_by_region`: Age (in seconds) of the oldest acknowledged message retained in a topic, broken down by Cloud region. +- `pubsub.topic.oldest_unacked_message_age_by_region`: Age (in seconds) of the oldest unacknowledged message in a topic, broken down by Cloud region. +- `pubsub.topic.retained_acked_bytes_by_region`: Total byte size of the acknowledged messages retained in a topic, broken down by Cloud region. - `pubsub.topic.send_message_operation_count`: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. - `pubsub.topic.send_request_count`: Cumulative count of publish requests, grouped by result. +- `pubsub.topic.unacked_bytes_by_region`: Total byte size of the unacknowledged messages in a topic, broken down by Cloud region. diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml index 64063c44497..32e4acbb521 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml +++ b/x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml @@ -55,6 +55,51 @@ - name: streaming_pull_response_count type: long description: Cumulative count of streaming pull responses, grouped by result. + - name: dead_letter_message_count + type: long + description: Cumulative count of messages published to dead letter topic, grouped by result. + - name: mod_ack_deadline_message_count + type: long + description: Cumulative count of messages whose deadline was updated by ModifyAckDeadline requests, grouped by delivery type. + - name: mod_ack_deadline_message_operation_count + type: long + description: Cumulative count of ModifyAckDeadline message operations, grouped by result. + - name: mod_ack_deadline_request_count + type: long + description: Cumulative count of ModifyAckDeadline requests, grouped by result. + - name: oldest_retained_acked_message_age + type: long + description: Age (in seconds) of the oldest acknowledged message retained in a subscription. + - name: oldest_retained_acked_message_age_by_region + type: long + description: Age (in seconds) of the oldest acknowledged message retained in a subscription, broken down by Cloud region. + - name: oldest_unacked_message_age_by_region + type: long + description: Age (in seconds) of the oldest unacknowledged message in a subscription, broken down by Cloud region. + - name: retained_acked_bytes + type: long + description: Total byte size of the acknowledged messages retained in a subscription. + - name: retained_acked_bytes_by_region + type: long + description: Total byte size of the acknowledged messages retained in a subscription, broken down by Cloud region. + - name: seek_request_count + type: long + description: Cumulative count of seek attempts, grouped by result. + - name: streaming_pull_mod_ack_deadline_message_operation_count + type: long + description: Cumulative count of StreamingPull ModifyAckDeadline operations, grouped by result. + - name: streaming_pull_mod_ack_deadline_request_count + type: long + description: Cumulative count of streaming pull requests with non-empty ModifyAckDeadline fields, grouped by result. + - name: byte_cost + type: long + description: Cumulative cost of operations, measured in bytes. This is used to measure quota utilization. + - name: config_updates_count + type: long + description: Cumulative count of configuration changes for each subscription, grouped by operation type and result. + - name: unacked_bytes_by_region + type: long + description: Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. - name: topic type: group description: Topic related metrics @@ -68,3 +113,46 @@ - name: send_request_count type: long description: Cumulative count of publish requests, grouped by result. + - name: oldest_retained_acked_message_age_by_region + type: long + description: Age (in seconds) of the oldest acknowledged message retained in a topic, broken down by Cloud region. + - name: oldest_unacked_message_age_by_region + type: long + description: Age (in seconds) of the oldest unacknowledged message in a topic, broken down by Cloud region. + - name: retained_acked_bytes_by_region + type: long + description: Total byte size of the acknowledged messages retained in a topic, broken down by Cloud region. + - name: byte_cost + type: long + description: Cost of operations, measured in bytes. This is used to measure utilization for quotas. + - name: config_updates_count + type: long + description: Cumulative count of configuration changes, grouped by operation type and result. + - name: unacked_bytes_by_region + type: long + description: Total byte size of the unacknowledged messages in a topic, broken down by Cloud region. + - name: snapshot + type: group + description: Snapshot related metrics + fields: + - name: oldest_message_age + type: long + description: Age (in seconds) of the oldest message retained in a snapshot. + - name: oldest_message_age_by_region + type: long + description: Age (in seconds) of the oldest message retained in a snapshot, broken down by Cloud region. + - name: backlog_bytes + type: long + description: Total byte size of the messages retained in a snapshot. + - name: backlog_bytes_by_region + type: long + description: Total byte size of the messages retained in a snapshot, broken down by Cloud region. + - name: num_messages + type: long + description: Number of messages retained in a snapshot. + - name: num_messages_by_region + type: long + description: Number of messages retained in a snapshot, broken down by Cloud region. + - name: config_updates_count + type: long + description: Cumulative count of configuration changes, grouped by operation type and result. diff --git a/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml index e7e3e032b71..3d8cdb0949c 100644 --- a/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml +++ b/x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml @@ -6,22 +6,50 @@ input: stackdriver: service: pubsub metrics: + - "pubsub.googleapis.com/snapshot/backlog_bytes" + - "pubsub.googleapis.com/snapshot/backlog_bytes_by_region" + - "pubsub.googleapis.com/snapshot/config_updates_count" + - "pubsub.googleapis.com/snapshot/num_messages" + - "pubsub.googleapis.com/snapshot/num_messages_by_region" + - "pubsub.googleapis.com/snapshot/oldest_message_age" + - "pubsub.googleapis.com/snapshot/oldest_message_age_by_region" - "pubsub.googleapis.com/subscription/ack_message_count" - "pubsub.googleapis.com/subscription/backlog_bytes" + - "pubsub.googleapis.com/subscription/byte_cost" + - "pubsub.googleapis.com/subscription/config_updates_count" + - "pubsub.googleapis.com/subscription/dead_letter_message_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_message_operation_count" + - "pubsub.googleapis.com/subscription/mod_ack_deadline_request_count" - "pubsub.googleapis.com/subscription/num_outstanding_messages" - "pubsub.googleapis.com/subscription/num_undelivered_messages" + - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age" + - "pubsub.googleapis.com/subscription/oldest_retained_acked_message_age_by_region" - "pubsub.googleapis.com/subscription/oldest_unacked_message_age" + - "pubsub.googleapis.com/subscription/oldest_unacked_message_age_by_region" - "pubsub.googleapis.com/subscription/pull_ack_message_operation_count" - "pubsub.googleapis.com/subscription/pull_ack_request_count" - "pubsub.googleapis.com/subscription/pull_message_operation_count" - "pubsub.googleapis.com/subscription/pull_request_count" - "pubsub.googleapis.com/subscription/push_request_count" - "pubsub.googleapis.com/subscription/push_request_latencies" + - "pubsub.googleapis.com/subscription/retained_acked_bytes" + - "pubsub.googleapis.com/subscription/retained_acked_bytes_by_region" + - "pubsub.googleapis.com/subscription/seek_request_count" - "pubsub.googleapis.com/subscription/sent_message_count" - "pubsub.googleapis.com/subscription/streaming_pull_ack_message_operation_count" - "pubsub.googleapis.com/subscription/streaming_pull_ack_request_count" - "pubsub.googleapis.com/subscription/streaming_pull_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_mod_ack_deadline_message_operation_count" + - "pubsub.googleapis.com/subscription/streaming_pull_mod_ack_deadline_request_count" - "pubsub.googleapis.com/subscription/streaming_pull_response_count" + - "pubsub.googleapis.com/subscription/unacked_bytes_by_region" + - "pubsub.googleapis.com/topic/byte_cost" + - "pubsub.googleapis.com/topic/config_updates_count" - "pubsub.googleapis.com/topic/message_sizes" + - "pubsub.googleapis.com/topic/oldest_retained_acked_message_age_by_region" + - "pubsub.googleapis.com/topic/oldest_unacked_message_age_by_region" + - "pubsub.googleapis.com/topic/retained_acked_bytes_by_region" - "pubsub.googleapis.com/topic/send_message_operation_count" - "pubsub.googleapis.com/topic/send_request_count" + - "pubsub.googleapis.com/topic/unacked_bytes_by_region"