Merge branch 'main' into retestsched

.github/CODEOWNERS

@@ -116,6 +116,8 @@ CHANGELOG*
 /x-pack/filebeat/input/gcs/ @elastic/security-external-integrations
 /x-pack/filebeat/input/http_endpoint/ @elastic/security-external-integrations
 /x-pack/filebeat/input/httpjson/ @elastic/security-external-integrations
+/x-pack/filebeat/input/internal/httplog @elastic/security-external-integrations
+/x-pack/filebeat/input/internal/httpmon @elastic/security-external-integrations
 /x-pack/filebeat/input/lumberjack/ @elastic/security-external-integrations
 /x-pack/filebeat/input/netflow/ @elastic/security-external-integrations
 /x-pack/filebeat/input/o365audit/ @elastic/security-external-integrations

CHANGELOG.next.asciidoc

@@ -125,6 +125,7 @@ https:/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Resolve statsd module's prematurely halting of metrics parsing upon encountering an invalid packet. {pull}35075[35075]
 - Fix the gap in fetching forecast API metrics at the end of each month for Azure billing module {pull}36142[36142]
 - Add option in SQL module to execute queries for all dbs. {pull}35688[35688]
+- Fix GCP compute metadata. {pull}36338[36338]
 - Add support for api_key authentication in elasticsearch module {pull}36274[36274]
 - Add remaining dimensions for azure storage account to make them available for tsdb enablement. {pull}36331[36331]
 - Add missing 'TransactionType' dimension for Azure Storage Account. {pull}36413[36413]
@@ -228,6 +229,7 @@ https:/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Migrate Azure Billing, Monitor, and Storage metricsets to the newer SDK. {pull}33585[33585]
 - Add support for float64 values parsing for statsd metrics of counter type. {pull}35099[35099]
 - Add kubernetes.deployment.status.* fields for Kubernetes module {pull}35999[35999]
+- Add Azure resource tags support to Azure Billing module {pull}36428[36428]
 
 
 *Osquerybeat*

NOTICE.txt

@@ -12712,11 +12712,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.3.11
+Version: v0.3.13
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].11/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].13/LICENSE:
 
  Apache License
  Version 2.0, January 2004
@@ -23293,11 +23293,11 @@ Contents of probable licence file $GOMODCACHE/github.com/xdg/[email protected]/LICENS
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm/module/apmelasticsearch/v2
-Version: v2.4.3
+Version: v2.4.4
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmelasticsearch/[email protected].3/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmelasticsearch/[email protected].4/LICENSE:
 
  Apache License
  Version 2.0, January 2004
@@ -23504,11 +23504,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmelasti
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm/module/apmhttp/v2
-Version: v2.4.3
+Version: v2.4.4
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmhttp/[email protected].3/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmhttp/[email protected].4/LICENSE:
 
  Apache License
  Version 2.0, January 2004
@@ -23715,11 +23715,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmhttp/v
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm/v2
-Version: v2.4.3
+Version: v2.4.4
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/[email protected].3/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/[email protected].4/LICENSE:
 
  Apache License
  Version 2.0, January 2004

filebeat/docs/modules/sonicwall.asciidoc

@@ -7,7 +7,7 @@ This file is generated! See scripts/docs_collector.py
 [[filebeat-module-sonicwall]]
 [role="xpack"]
 
-:modulename: sonicwall
+:modulename: sonicwall_firewall
 :has-dashboards: false
 
 == Sonicwall module

filebeat/input/internal/procnet/procnet_test.go

@@ -30,7 +30,7 @@ func TestAddrs(t *testing.T) {
  t.Errorf("unexpected error: %v", err)
  }
  if len(addr4) == 0 {
- t.Errorf("expected addr in addr4 for IPv4 address: %v", addr6)
+ t.Errorf("expected addr in addr4 for IPv4 address: addr6 is %v", addr6)
  }
  if len(addr6) != 0 {
  t.Errorf("unexpected addrs in addr6 for IPv4 address: %v", addr6)
@@ -43,10 +43,10 @@ func TestAddrs(t *testing.T) {
  t.Errorf("unexpected error: %v", err)
  }
  if len(addr4) != 0 {
- t.Errorf("expected addr in addr4 for IPv4 address: %v", addr6)
+ t.Errorf("unexpected addr in addr4 for IPv6 address: %v", addr4)
  }
  if len(addr6) == 0 {
- t.Errorf("unexpected addrs in addr6 for IPv4 address: %v", addr6)
+ t.Errorf("expected addrs in addr6 for IPv6 address: addr4 is %v", addr4)
  }
  })
 }

go.mod

@@ -202,7 +202,7 @@ require (
  github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
  github.com/elastic/bayeux v1.0.5
  github.com/elastic/elastic-agent-autodiscover v0.6.2
- github.com/elastic/elastic-agent-libs v0.3.11
+ github.com/elastic/elastic-agent-libs v0.3.13
  github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
  github.com/elastic/elastic-agent-system-metrics v0.6.1
  github.com/elastic/go-elasticsearch/v8 v8.9.0
@@ -218,9 +218,9 @@ require (
  github.com/pierrec/lz4/v4 v4.1.16
  github.com/pkg/xattr v0.4.9
  github.com/shirou/gopsutil/v3 v3.22.10
- go.elastic.co/apm/module/apmelasticsearch/v2 v2.4.3
- go.elastic.co/apm/module/apmhttp/v2 v2.4.3
- go.elastic.co/apm/v2 v2.4.3
+ go.elastic.co/apm/module/apmelasticsearch/v2 v2.4.4
+ go.elastic.co/apm/module/apmhttp/v2 v2.4.4
+ go.elastic.co/apm/v2 v2.4.4
  go.mongodb.org/mongo-driver v1.5.1
  gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )

go.sum

@@ -653,8 +653,8 @@ github.com/elastic/elastic-agent-autodiscover v0.6.2 h1:7P3cbMBWXjbzA80rxitQjc+P
 github.com/elastic/elastic-agent-autodiscover v0.6.2/go.mod h1:yXYKFAG+Py+TcE4CCR8EAbJiYb+6Dz9sCDoWgOveqtU=
 github.com/elastic/elastic-agent-client/v7 v7.3.0 h1:LugKtBXK7bp4SFL/uQqGU/f4Ppx12Jk5a36voGabLa0=
 github.com/elastic/elastic-agent-client/v7 v7.3.0/go.mod h1:9/amG2K2y2oqx39zURcc+hnqcX+nyJ1cZrLgzsgo5c0=
-github.com/elastic/elastic-agent-libs v0.3.11 h1:FupAciF/QjrlDMZQkQXwcurJeMRZnN4HOG7QDJ8XYwA=
-github.com/elastic/elastic-agent-libs v0.3.11/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ=
+github.com/elastic/elastic-agent-libs v0.3.13 h1:qFiBWeBfjsBId+i31rggyW2ZjzA9qBRz7wIiy+rkcvc=
+github.com/elastic/elastic-agent-libs v0.3.13/go.mod h1:mpSfrigixx8x+uMxWKl4LtdlrKIhZbA4yT2eIeIazUQ=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U=
 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI=
 github.com/elastic/elastic-agent-system-metrics v0.6.1 h1:LCN1lvQTkdUuU/rKlpKyVMDU/G/I8/iZWCaW6K+mo4o=
@@ -1896,12 +1896,12 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
 github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
 github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
 github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
-go.elastic.co/apm/module/apmelasticsearch/v2 v2.4.3 h1:ahFfYDBPNq6G0Vsqncg3cs0HwcEQ76yKdhQeKKvGPck=
-go.elastic.co/apm/module/apmelasticsearch/v2 v2.4.3/go.mod h1:2kByxP0WjwX/9vRTKGAlqctTr7tQLN7hSfWPNLQJ+eM=
-go.elastic.co/apm/module/apmhttp/v2 v2.4.3 h1:bBqbbtQSEL+uVpH5CS656E9x6pXha8kkZ468/G0T5Eo=
-go.elastic.co/apm/module/apmhttp/v2 v2.4.3/go.mod h1:PsafA29/iwRva1ouV+70Lfu6RyAJHh6Zcc+jL6eZ83E=
-go.elastic.co/apm/v2 v2.4.3 h1:k6mj63O7IIyqqn3S52C2vBXvaSK9M5FHp0aZHpPH/as=
-go.elastic.co/apm/v2 v2.4.3/go.mod h1:+CiBUdrrAGnGCL9TNx7tQz3BrfYV23L8Ljvotoc87so=
+go.elastic.co/apm/module/apmelasticsearch/v2 v2.4.4 h1:TVcEjy/gs/DVk3aaBYNGm6B7uOkC6BTlDFRzyvBkMlw=
+go.elastic.co/apm/module/apmelasticsearch/v2 v2.4.4/go.mod h1:xWKnT5lKqFwNgT3nkWumBCXCt597cHkSuyIWWMXpv78=
+go.elastic.co/apm/module/apmhttp/v2 v2.4.4 h1:BcJDLFztZByHiJRXXvAHY7bue9+vhzMLn2OBK5N8HDw=
+go.elastic.co/apm/module/apmhttp/v2 v2.4.4/go.mod h1:694qsHWRPVkfYqQ224ZSeRx9Dt2FS/xF1igNBGkHi6o=
+go.elastic.co/apm/v2 v2.4.4 h1:KGbmUEtLNQ6sCFGpqTMhfRjqDOKP8IliO+efErKY5lU=
+go.elastic.co/apm/v2 v2.4.4/go.mod h1:+CiBUdrrAGnGCL9TNx7tQz3BrfYV23L8Ljvotoc87so=
 go.elastic.co/ecszap v1.0.1 h1:mBxqEJAEXBlpi5+scXdzL7LTFGogbuxipJC0KTZicyA=
 go.elastic.co/ecszap v1.0.1/go.mod h1:SVjazT+QgNeHSGOCUHvRgN+ZRj5FkB7IXQQsncdF57A=
 go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4=

testing/environments/snapshot.yml

@@ -3,7 +3,7 @@
 version: '2.3'
 services:
  elasticsearch:
- image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0-76b03348-SNAPSHOT
+ image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0-16180d4a-SNAPSHOT
  # When extend is used it merges healthcheck.tests, see:
  # https:/docker/compose/issues/8962
  # healthcheck:
@@ -31,7 +31,7 @@ services:
  - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles"
 
  logstash:
- image: docker.elastic.co/logstash/logstash:8.11.0-76b03348-SNAPSHOT
+ image: docker.elastic.co/logstash/logstash:8.11.0-16180d4a-SNAPSHOT
  healthcheck:
  test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"]
  retries: 600
@@ -44,7 +44,7 @@ services:
  - 5055:5055
 
  kibana:
- image: docker.elastic.co/kibana/kibana:8.11.0-76b03348-SNAPSHOT
+ image: docker.elastic.co/kibana/kibana:8.11.0-16180d4a-SNAPSHOT
  environment:
  - "ELASTICSEARCH_USERNAME=kibana_system_user"
  - "ELASTICSEARCH_PASSWORD=testing"

x-pack/filebeat/input/httpjson/config_request.go

@@ -151,7 +151,7 @@ func (c *requestConfig) Validate() error {
  return fmt.Errorf("unsupported method %q", c.Method)
  }
 
- if _, err := newBasicTransformsFromConfig(c.Transforms, requestNamespace, nil); err != nil {
+ if _, err := newBasicTransformsFromConfig(registeredTransforms, c.Transforms, requestNamespace, nil); err != nil {
  return err
  }
 

x-pack/filebeat/input/httpjson/config_response.go

@@ -37,10 +37,10 @@ type splitConfig struct {
 }
 
 func (c *responseConfig) Validate() error {
- if _, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace, nil); err != nil {
+ if _, err := newBasicTransformsFromConfig(registeredTransforms, c.Transforms, responseNamespace, nil); err != nil {
  return err
  }
- if _, err := newBasicTransformsFromConfig(c.Pagination, paginationNamespace, nil); err != nil {
+ if _, err := newBasicTransformsFromConfig(registeredTransforms, c.Pagination, paginationNamespace, nil); err != nil {
  return err
  }
  if c.DecodeAs != "" {
@@ -52,7 +52,7 @@ func (c *responseConfig) Validate() error {
 }
 
 func (c *splitConfig) Validate() error {
- if _, err := newBasicTransformsFromConfig(c.Transforms, responseNamespace, nil); err != nil {
+ if _, err := newBasicTransformsFromConfig(registeredTransforms, c.Transforms, responseNamespace, nil); err != nil {
  return err
  }
 

x-pack/filebeat/input/httpjson/encoding.go

@@ -14,57 +14,8 @@ import (
  "net/http"
 
  "github.com/elastic/mito/lib/xml"
-
- "github.com/elastic/elastic-agent-libs/logp"
-)
-
-type encoderFunc func(trReq transformable) ([]byte, error)
-
-type decoderFunc func(p []byte, dst *response) error
-
-var (
- registeredEncoders = map[string]encoderFunc{}
- registeredDecoders = map[string]decoderFunc{}
- defaultEncoder encoderFunc = encodeAsJSON
- defaultDecoder decoderFunc = decodeAsJSON
 )
 
-func registerEncoder(contentType string, enc encoderFunc) error {
- if contentType == "" {
- return errors.New("content-type can't be empty")
- }
-
- if enc == nil {
- return errors.New("encoder can't be nil")
- }
-
- if _, found := registeredEncoders[contentType]; found {
- return errors.New("already registered")
- }
-
- registeredEncoders[contentType] = enc
-
- return nil
-}
-
-func registerDecoder(contentType string, dec decoderFunc) error {
- if contentType == "" {
- return errors.New("content-type can't be empty")
- }
-
- if dec == nil {
- return errors.New("decoder can't be nil")
- }
-
- if _, found := registeredDecoders[contentType]; found {
- return errors.New("already registered")
- }
-
- registeredDecoders[contentType] = dec
-
- return nil
-}
-
 func encode(contentType string, trReq transformable) ([]byte, error) {
  enc, found := registeredEncoders[contentType]
  if !found {
@@ -81,35 +32,34 @@ func decode(contentType string, p []byte, dst *response) error {
  return dec(p, dst)
 }
 
-func registerEncoders() {
- log := logp.L().Named(logName)
- log.Debugf("registering encoder 'application/json': returned error: %#v",
- registerEncoder("application/json", encodeAsJSON))
-
- log.Debugf("registering encoder 'application/x-www-form-urlencoded': returned error: %#v",
- registerEncoder("application/x-www-form-urlencoded", encodeAsForm))
-}
-
-func registerDecoders() {
- log := logp.L().Named(logName)
- log.Debugf("registering decoder 'application/json': returned error: %#v",
- registerDecoder("application/json", decodeAsJSON))
-
- log.Debugf("registering decoder 'application/x-ndjson': returned error: %#v",
- registerDecoder("application/x-ndjson", decodeAsNdjson))
-
- log.Debugf("registering decoder 'text/csv': returned error: %#v",
- registerDecoder("text/csv", decodeAsCSV))
-
- log.Debugf("registering decoder 'application/zip': returned error: %#v",
- registerDecoder("application/zip", decodeAsZip))
+var (
+ // registeredEncoders is the set of available encoders.
+ registeredEncoders = map[string]encoderFunc{
+ "application/json": encodeAsJSON,
+ "application/x-www-form-urlencoded": encodeAsForm,
+ }
+ // defaultEncoder is the decoder used when no registers
+ // encoder is available.
+ defaultEncoder = encodeAsJSON
+
+ // registeredDecoders is the set of available decoders.
+ registeredDecoders = map[string]decoderFunc{
+ "application/json": decodeAsJSON,
+ "application/x-ndjson": decodeAsNdjson,
+ "text/csv": decodeAsCSV,
+ "application/zip": decodeAsZip,
+ "application/xml": decodeAsXML,
+ "text/xml; charset=utf-8": decodeAsXML,
+ }
+ // defaultDecoder is the decoder used when no registers
+ // decoder is available.
+ defaultDecoder = decodeAsJSON
+)
 
- log.Debugf("registering decoder 'application/xml': returned error: %#v",
- registerDecoder("application/xml", decodeAsXML))
- log.Debugf("registering decoder 'text/xml': returned error: %#v",
- registerDecoder("text/xml; charset=utf-8", decodeAsXML))
-}
+type encoderFunc func(trReq transformable) ([]byte, error)
+type decoderFunc func(p []byte, dst *response) error
 
+// encodeAsJSON encodes trReq as a JSON message.
 func encodeAsJSON(trReq transformable) ([]byte, error) {
  if len(trReq.body()) == 0 {
  return nil, nil
@@ -120,10 +70,12 @@ func encodeAsJSON(trReq transformable) ([]byte, error) {
  return json.Marshal(trReq.body())
 }
 
+// decodeAsJSON decodes the JSON message in p into dst.
 func decodeAsJSON(p []byte, dst *response) error {
  return json.Unmarshal(p, &dst.body)
 }
 
+// encodeAsForm encodes trReq as a URL encoded form.
 func encodeAsForm(trReq transformable) ([]byte, error) {
  url := trReq.url()
  body := []byte(url.RawQuery)
@@ -135,6 +87,8 @@ func encodeAsForm(trReq transformable) ([]byte, error) {
  return body, nil
 }
 
+// decodeAsNdjson decodes the message in p as a JSON object stream
+// It is more relaxed than NDJSON.
 func decodeAsNdjson(p []byte, dst *response) error {
  var results []interface{}
  dec := json.NewDecoder(bytes.NewReader(p))
@@ -149,6 +103,7 @@ func decodeAsNdjson(p []byte, dst *response) error {
  return nil
 }
 
+// decodeAsCSV decodes p as a headed CSV document into dst.
 func decodeAsCSV(p []byte, dst *response) error {
  var results []interface{}
 
@@ -189,6 +144,7 @@ func decodeAsCSV(p []byte, dst *response) error {
  return nil
 }
 
+// decodeAsZip decodes p as a ZIP archive into dst.
 func decodeAsZip(p []byte, dst *response) error {
  var results []interface{}
  r, err := zip.NewReader(bytes.NewReader(p), int64(len(p)))
@@ -225,6 +181,7 @@ func decodeAsZip(p []byte, dst *response) error {
  return nil
 }
 
+// decodeAsXML decodes p as an XML document into dst.
 func decodeAsXML(p []byte, dst *response) error {
  cdata, body, err := xml.Unmarshal(bytes.NewReader(p), dst.xmlDetails)
  if err != nil {