Write to .monitoring index on elasticsearch if xpack.enabled is true …

…on standalone metricbeat (#28365)

CHANGELOG.next.asciidoc

@@ -227,6 +227,7 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - `beat` module respects `basepath` config option. {pull}28162[28162]
 - Fix list_docker.go {pull}28374[28374]
 - Divide RDS metric cpu.total.pct by 100. {pull}28456[28456]
+- Use xpack.enabled on SM modules to write into .monitoring indices when using Metricbeat standalone {pull}28365[28365]
 
 *Packetbeat*
 

metricbeat/module/beat/config.go

@@ -0,0 +1,30 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package beat
+
+// Config defines the structure for the Beat module configuration options
+type Config struct {
+ XPackEnabled bool `config:"xpack.enabled"`
+}
+
+// DefaultConfig returns the default configuration for the Beat module
+func DefaultConfig() Config {
+ return Config{
+ XPackEnabled: false,
+ }
+}

metricbeat/module/beat/metricset.go

@@ -26,11 +26,17 @@ import (
 type MetricSet struct {
  mb.BaseMetricSet
  *helper.HTTP
+ XPackEnabled bool
 }
 
 // NewMetricSet creates a metricset that can be used to build other metricsets
 // within the Beat module.
 func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
+ config := DefaultConfig()
+ if err := base.Module().UnpackConfig(&config); err != nil {
+ return nil, err
+ }
+
  http, err := helper.NewHTTP(base)
  if err != nil {
  return nil, err
@@ -39,6 +45,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
  ms := &MetricSet{
  base,
  http,
+ config.XPackEnabled,
  }
 
  return ms, nil

metricbeat/module/beat/state/_meta/data.json

@@ -1,77 +1,61 @@
 {
- "@timestamp": "2021-03-30T18:16:03.880Z",
- "@metadata": {
- "beat": "metricbeat",
- "type": "_doc",
- "version": "8.0.0"
- },
- "agent": {
- "version": "8.0.0",
- "ephemeral_id": "ed46455e-9adf-4493-83ba-a145ad0371a9",
- "id": "7a0a00bb-faa3-4b2b-9d92-4a0a17bcb8f3",
- "name": "anonymous",
- "type": "metricbeat"
- },
- "ecs": {
- "version": "1.8.0"
- },
- "metricset": {
- "name": "state",
- "period": 10000
- },
+ "@timestamp": "2017-10-12T08:05:34.853Z",
  "beat": {
  "state": {
  "beat": {
- "name": "anonymous",
- "host": "anonymous",
+ "host": "2963d991095f",
+ "name": "2963d991095f",
  "type": "metricbeat",
- "uuid": "1be2b779-2479-44c4-9027-2e6fcc0cee93",
- "version": "8.0.0"
- },
- "module": {
- "count": 10
+ "uuid": "c4c9bc08-e990-4529-8bf5-715c00fa6615",
+ "version": "7.12.0"
  },
- "output": {
- "name": "elasticsearch"
- },
- "queue": {
- "name": "mem"
+ "cluster": {
+ "uuid": "foobar"
  },
  "host": {
  "containerized": "containerized",
  "os": {
- "platform": "antergos",
- "version": "",
- "kernel": "5.11.7-arch1-1",
- "name": "Antergos Linux"
+ "kernel": "5.14.13-200.fc34.x86_64",
+ "name": "CentOS Linux",
+ "platform": "centos",
+ "version": "7 (Core)"
  }
  },
  "management": {
  "enabled": false
  },
+ "module": {
+ "count": 10
+ },
+ "output": {
+ "name": "elasticsearch"
+ },
+ "queue": {
+ "name": "mem"
+ },
  "service": {
- "id": "1be2b779-2479-44c4-9027-2e6fcc0cee93",
+ "id": "c4c9bc08-e990-4529-8bf5-715c00fa6615",
  "name": "metricbeat",
- "version": "8.0.0"
- },
- "cluster": {
- "uuid": "-6NL06F2Td6bMGdkyaJ0PA"
+ "version": "7.12.0"
  }
  }
  },
+ "event": {
+ "dataset": "beat.state",
+ "duration": 115000,
+ "module": "beat"
+ },
  "host": {
- "name": "anonymous",
  "architecture": "x86_64",
- "hostname": "anonymous",
- "id": "54f70115bae545cbac2b150f254472a0"
+ "hostname": "2963d991095f",
+ "id": "44ccc339d95bd51386fcfc5d8f041927"
+ },
+ "metricset": {
+ "name": "state",
+ "period": 10000
  },
  "service": {
- "address": "http://localhost:5066/state",
+ "address": "172.19.0.2:5066",
  "type": "beat"
- },
- "event": {
- "module": "beat",
- "duration": 4263542,
- "dataset": "beat.state"
  }
-}
+}

metricbeat/module/beat/state/data.go

@@ -20,6 +20,8 @@ package state
 import (
  "encoding/json"
 
+ "github.com/elastic/beats/v7/metricbeat/helper/elastic"
+
  "github.com/elastic/beats/v7/libbeat/common"
 
  "github.com/pkg/errors"
@@ -65,7 +67,7 @@ var (
  }
 )
 
-func eventMapping(r mb.ReporterV2, info beat.Info, content []byte) error {
+func eventMapping(r mb.ReporterV2, info beat.Info, content []byte, isXpack bool) error {
  event := mb.Event{
  RootFields: common.MapStr{},
  }
@@ -135,6 +137,13 @@ func eventMapping(r mb.ReporterV2, info beat.Info, content []byte) error {
  event.MetricSetFields["host"] = hostMap
  }
 
+ // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*`
+ // When using Agent, the index name is overwritten anyways.
+ if isXpack {
+ index := elastic.MakeXPackMonitoringIndexName(elastic.Beats)
+ event.Index = index
+ }
+
  r.Event(event)
 
  return nil

metricbeat/module/beat/state/data_test.go

@@ -47,7 +47,7 @@ func TestEventMapping(t *testing.T) {
  require.NoError(t, err)
 
  reporter := &mbtest.CapturingReporterV2{}
- err = eventMapping(reporter, info, input)
+ err = eventMapping(reporter, info, input, true)
 
  require.NoError(t, err, f)
  require.True(t, len(reporter.GetEvents()) >= 1, f)

metricbeat/module/beat/state/state.go

@@ -66,5 +66,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
  return err
  }
 
- return eventMapping(r, *info, content)
+ return eventMapping(r, *info, content, m.XPackEnabled)
 }