Add /inputs/ route for Filebeat input metrics

Expose metrics from Filebeat inputs under /inputs/. This moves metrics for aws-s3, aws-cloudwatch,
cel, and lumberjack that were previously registered under the "dataset" monitoring namespace to the
"inputs" namespace.

`/inputs/` returns metrics related to input instances. It returns a list of
objects where each object contains metrics for an instance of an input. Each
object will minimally contain an `input` field that identifies the type of input
(e.g. `aws-s3`) and an `id` field that is the unique identifier for the input
instance.

A request may optionally specify a `type` query parameter to request metrics
for a specific type of input. And `pretty` may be included to have the
returned JSON be pretty formatted.

CHANGELOG.next.asciidoc

@@ -17,6 +17,7 @@ https:/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 *Filebeat*
 
 - Fixed error spam from `add_kubernetes_metadata` processor when running on AKS. {pull}33697[33697]
+- Metrics hosted by the HTTP monitoring endpoint for the `aws-cloudwatch`, `aws-s3`, `cel`, and `lumberjack` inputs are now available under `/inputs/` instead of `/dataset`.
 
 *Heartbeat*
 

NOTICE.txt

@@ -15758,6 +15758,75 @@ Contents of probable licence file $GOMODCACHE/github.com/gorhill/[email protected]
  limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/gorilla/handlers
+Version: v1.5.1
+Licence type (autodetected): BSD-2-Clause
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/gorilla/[email protected]/LICENSE:
+
+Copyright (c) 2013 The Gorilla Handlers Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+ Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+--------------------------------------------------------------------------------
+Dependency : github.com/gorilla/mux
+Version: v1.8.0
+Licence type (autodetected): BSD-3-Clause
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/gorilla/[email protected]/LICENSE:
+
+Copyright (c) 2012-2018 The Gorilla Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/h2non/filetype
 Version: v1.1.1
@@ -33082,6 +33151,35 @@ Contents of probable licence file $GOMODCACHE/github.com/fearful-symmetry/gomsr@
  limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/felixge/httpsnoop
+Version: v1.0.1
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/felixge/[email protected]/LICENSE.txt:
+
+Copyright (c) 2016 Felix Geisendörfer ([email protected])
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/form3tech-oss/jwt-go
 Version: v3.2.3+incompatible
@@ -35765,43 +35863,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
---------------------------------------------------------------------------------
-Dependency : github.com/gorilla/mux
-Version: v1.8.0
-Licence type (autodetected): BSD-3-Clause
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/github.com/gorilla/[email protected]/LICENSE:
-
-Copyright (c) 2012-2018 The Gorilla Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
- * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 --------------------------------------------------------------------------------
 Dependency : github.com/gorilla/websocket
 Version: v1.4.2

filebeat/beater/filebeat.go

@@ -38,6 +38,7 @@ import (
  "github.com/elastic/beats/v7/libbeat/common/reload"
  "github.com/elastic/beats/v7/libbeat/esleg/eslegclient"
  "github.com/elastic/beats/v7/libbeat/management"
+ "github.com/elastic/beats/v7/libbeat/monitoring/inputmon"
  "github.com/elastic/beats/v7/libbeat/outputs/elasticsearch"
  "github.com/elastic/beats/v7/libbeat/publisher/pipetool"
  "github.com/elastic/beats/v7/libbeat/statestore"
@@ -115,6 +116,12 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea
  return nil, err
  }
 
+ if b.API != nil {
+ if err = inputmon.AttachHandler(b.API.Router()); err != nil {
+ return nil, fmt.Errorf("failed attach inputs api to monitoring endpoint server: %w", err)
+ }
+ }
+
  // Add inputs created by the modules
  config.Inputs = append(config.Inputs, moduleInputs...)
 

filebeat/docs/index.asciidoc

@@ -23,6 +23,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
 :has_kubernetes_logs_path_matcher:
 :has_nomad_logs_path_matcher:
 :has_registry:
+:has_inputs_endpoint:
 :deb_os:
 :rpm_os:
 :mac_os:

go.mod

@@ -201,6 +201,8 @@ require (
  github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640
  github.com/google/cel-go v0.12.5
  github.com/googleapis/gax-go/v2 v2.5.1
+ github.com/gorilla/handlers v1.5.1
+ github.com/gorilla/mux v1.8.0
  github.com/pierrec/lz4/v4 v4.1.15
  github.com/shirou/gopsutil/v3 v3.21.12
  go.elastic.co/apm/module/apmelasticsearch/v2 v2.0.0
@@ -257,6 +259,7 @@ require (
  github.com/elastic/go-windows v1.0.1 // indirect
  github.com/evanphx/json-patch v4.12.0+incompatible // indirect
  github.com/fearful-symmetry/gomsr v0.0.1 // indirect
+ github.com/felixge/httpsnoop v1.0.1 // indirect
  github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
  github.com/go-logfmt/logfmt v0.5.1 // indirect
  github.com/go-logr/logr v1.2.3 // indirect

go.sum

@@ -705,6 +705,8 @@ github.com/fearful-symmetry/gomsr v0.0.1 h1:m208RzdTApWVbv8a9kf78rdPLQe+BY9AxRb/
 github.com/fearful-symmetry/gomsr v0.0.1/go.mod h1:Qb/0Y7zwobP7v8Sji+M5mlL4N7Voyz5WaKXXRFPnLio=
 github.com/fearful-symmetry/gorapl v0.0.4 h1:TMn4fhhtIAd+C3NrAl638oaYlX1vgcKNVVdad53oyiE=
 github.com/fearful-symmetry/gorapl v0.0.4/go.mod h1:XoeZ+5v0tJX9WMvzqdPaaKAdX7y17mDN3pxDGemINR0=
+github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
+github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c=
@@ -1050,6 +1052,8 @@ github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVf
 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
+github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
+github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=

libbeat/api/config.go

@@ -28,14 +28,12 @@ type Config struct {
  SecurityDescriptor string `config:"named_pipe.security_descriptor"`
 }
 
-var (
- // DefaultConfig is the default configuration used by the API endpoint.
- DefaultConfig = Config{
- Enabled: false,
- Host: "localhost",
- Port: 5066,
- }
-)
+// DefaultConfig is the default configuration used by the API endpoint.
+var DefaultConfig = Config{
+ Enabled: false,
+ Host: "localhost",
+ Port: 5066,
+}
 
 // File mode for the socket file, owner of the process can do everything, member of the group can read.
-const socketFileMode = os.FileMode(0740)
+const socketFileMode = os.FileMode(0o740)

libbeat/api/routes.go

@@ -22,33 +22,37 @@ import (
  "net/http"
  "net/url"
 
+ "go.uber.org/multierr"
+
  "github.com/elastic/elastic-agent-libs/config"
  "github.com/elastic/elastic-agent-libs/logp"
  "github.com/elastic/elastic-agent-libs/mapstr"
  "github.com/elastic/elastic-agent-libs/monitoring"
 )
 
-type handlerFunc func(http.ResponseWriter, *http.Request)
 type lookupFunc func(string) *monitoring.Namespace
 
-var handlerFuncMap = make(map[string]handlerFunc)
-
 // NewWithDefaultRoutes creates a new server with default API routes.
 func NewWithDefaultRoutes(log *logp.Logger, config *config.C, ns lookupFunc) (*Server, error) {
- mux := http.NewServeMux()
-
- mux.HandleFunc("/", makeRootAPIHandler(makeAPIHandler(ns("info"))))
- mux.HandleFunc("/state", makeAPIHandler(ns("state")))
- mux.HandleFunc("/stats", makeAPIHandler(ns("stats")))
- mux.HandleFunc("/dataset", makeAPIHandler(ns("dataset")))
+ api, err := New(log, config)
+ if err != nil {
+ return nil, err
+ }
 
- for api, h := range handlerFuncMap {
- mux.HandleFunc(api, h)
+ err = multierr.Combine(
+ api.AttachHandler("/", makeRootAPIHandler(makeAPIHandler(ns("info")))),
+ api.AttachHandler("/state", makeAPIHandler(ns("state"))),
+ api.AttachHandler("/stats", makeAPIHandler(ns("stats"))),
+ api.AttachHandler("/dataset", makeAPIHandler(ns("dataset"))),
+ )
+ if err != nil {
+ return nil, err
  }
- return New(log, mux, config)
+
+ return api, nil
 }
 
-func makeRootAPIHandler(handler handlerFunc) handlerFunc {
+func makeRootAPIHandler(handler http.HandlerFunc) http.HandlerFunc {
  return func(w http.ResponseWriter, r *http.Request) {
  if r.URL.Path != "/" {
  http.NotFound(w, r)
@@ -58,7 +62,7 @@ func makeRootAPIHandler(handler handlerFunc) handlerFunc {
  }
 }
 
-func makeAPIHandler(ns *monitoring.Namespace) handlerFunc {
+func makeAPIHandler(ns *monitoring.Namespace) http.HandlerFunc {
  return func(w http.ResponseWriter, r *http.Request) {
  w.Header().Set("Content-Type", "application/json; charset=utf-8")
 
@@ -80,12 +84,3 @@ func prettyPrint(w http.ResponseWriter, data mapstr.M, u *url.URL) {
  fmt.Fprint(w, data.String())
  }
 }
-
-// AddHandlerFunc provides interface to add customized handlerFunc
-func AddHandlerFunc(api string, h handlerFunc) error {
- if _, exist := handlerFuncMap[api]; exist {
- return fmt.Errorf("%s already exist", api)
- }
- handlerFuncMap[api] = h
- return nil
-}