Merge remote-tracking branch 'upstream/master' into deployment_name

.ci/packaging.groovy

@@ -246,8 +246,7 @@ pipeline {
  agent { label 'ubuntu-18 && immutable' }
  options { skipDefaultCheckout() }
  steps {
- log(level: 'WARN', text: "E2E Tests for Beats are disabled until latest breaking changes in Kibana affecting Package Registry are resolved.")
- //runE2ETests()
+ runE2ETests()
  }
  }
  }

CHANGELOG-developer.next.asciidoc

@@ -56,6 +56,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
 - Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
 - Removed the `common.Float` type. {issue}28279[28279] {pull}28280[28280] {pull}28376[28376]
+- libbeat.logp package forces ECS compliant logs. Logs are JSON formatted. Options to enable ECS/JSON have been removed. {issue}15544[15544] {pull}28573[28573]
 
 ==== Bugfixes
 
@@ -129,3 +130,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 ==== Deprecated
 
 - Deprecated the `common.Float` type. {issue}28279[28279] {pull}28280[28280]
+- Deprecate Beat generators. {pull}28814[28814]

CHANGELOG.next.asciidoc

@@ -20,9 +20,12 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve stats API {pull}27963[27963]
 - Enable IMDSv2 support for `add_cloud_metadata` processor on AWS. {issue}22101[22101] {pull}28285[28285]
 - Update kubernetes.namespace from keyword to group field and add name, labels, annotations, uuid as its fields {pull}27917[27917]
+- Libbeat: logp package forces ECS compliant logs. Logs are JSON formatted. Options to enable ECS/JSON have been removed. {issue}15544[15544] {pull}28573[28573]
 - Previously, RE2 and thus Golang had a bug where `(|a)*` matched more characters than `(|a)+`. To stay consistent with PCRE, the bug was fixed. Configurations that rely on the old, buggy behaviour has to be adjusted. See more about Golang bug: https:/golang/go/issues/46123 {pull}27543[27543]
 - Update docker client. {pull}28716[28716]
 - Remove `auto` from the available options of `setup.ilm.enabled` and set the default value to `true`. {pull}28671[28671]
+- add_process_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
+- add_docker_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
 
 *Auditbeat*
 
@@ -32,6 +35,7 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Auditd dataset: Removes the authentication_success and authentication_failure event.type values for user logins. {issue}19039[19039] {pull}28378[28378]
 - Fix handling of long file names on Windows. {issue}25334[25334] {pull}28517[28517]
 - System/socket dataset: Fix uninstallation of return kprobes. {issue}28608[28608] {pull}28609[28609]
+- Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
 
 *Filebeat*
 
@@ -53,6 +57,8 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove old `httpjson` config implementation. {pull}28054[28054]
 - Added dataset `threatq` to the `threatintel` module to ingest indicators from ThreatQ {issue}27423[27423]
 - Fail to start Filebat if none between `queue_url`, `bucket_arn` or `non_aws_bucket_name` is set for a configured aws-s3 input {issue}13911[13911] {pull}28666[28666]
+- All modules: Replace usages of deprecated ECS fields `process.ppid` and `log.original` with `process.parent.pid` and `event.original`. {pull}28620[28620]
+- Replace usages of `host.user.*` fields with `user.*` in `cisco`, `microsoft` and `oracle` modules. {pull}28620[28620]
 
 *Heartbeat*
 
@@ -75,12 +81,14 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated config option perfmon.counters from windows/perfmon metricset. {pull}28282[28282]
 - Remove deprecated fields in Redis module. {issue}11835[11835] {pull}28246[28246]
 - Align fields to Beats naming conventions in GCP module. {issue}27231[27231] {pull}27974[27974]
+- system/process metricset: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
 
 *Packetbeat*
 
 - Redis: fix incorrectly handle with two-words redis command. {issue}14872[14872] {pull}14873[14873]
 - `event.category` no longer contains the value `network_traffic` because this is not a valid ECS event category value. {pull}20556[20556]
 - Remove deprecated TLS fields in favor of tls.server.x509 and tls.client.x509 ECS fields. {pull}28487[28487]
+- HTTP: The field `http.request.method` will maintain its original case. {pull}28620[28620]
 
 *Winlogbeat*
 
@@ -221,6 +229,7 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - `beat` module respects `basepath` config option. {pull}28162[28162]
 - Fix list_docker.go {pull}28374[28374]
 - Divide RDS metric cpu.total.pct by 100. {pull}28456[28456]
+- Use xpack.enabled on SM modules to write into .monitoring indices when using Metricbeat standalone {pull}28365[28365]
 
 *Packetbeat*
 
@@ -270,6 +279,7 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Name all k8s workqueue. {pull}28085[28085]
 - Update kubernetes scheduler and controllermanager endpoints in elastic-agent-standalone-kubernetes.yaml with secure ports {pull}28675[28675]
 - Add options to configure k8s client qps/burst. {pull}28151[28151]
+- Update to ECS 8.0 fields. {pull}28620[28620]
 
 *Auditbeat*
 
@@ -370,6 +380,7 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Deprecate `log` input in favour of `filestream` input. {pull}28623[28623]
 
 *Heartbeat*
 

Jenkinsfile

@@ -617,8 +617,7 @@ def target(Map args = [:]) {
  pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch)
  }
  if(isE2E) {
- log(level: 'WARN', text: "E2E Tests for Beats are disabled until latest breaking changes in Kibana affecting Package Registry are resolved.")
- //e2e(args)
+ e2e(args)
  }
  }
  }
@@ -830,20 +829,23 @@ def archiveTestOutput(Map args = [:]) {
  catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
  withMageEnv(version: "${env.GO_VERSION}"){
  dir(directory){
- cmd(label: "Archive system tests files", script: 'mage packageSystemTests')
+ cmd(label: "Archive system tests files", script: 'mage packageSystemTests', returnStatus: true)
  }
  }
+
  def fileName = 'build/system-tests-*.tar.gz' // see dev-tools/mage/target/common/package.go#PackageSystemTests method
  def files = findFiles(glob: "${fileName}")
- files.each { file ->
- echo "${file.name}"
+
+ if (files?.length() > 0) {
+ googleStorageUploadExt(
+ bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
+ credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
+ pattern: "${fileName}",
+ sharedPublicly: true
+ )
+ } else {
+ log(level: 'WARN', text: "There are no system-tests files to upload Google Storage}")
  }
- googleStorageUploadExt(
- bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
- credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
- pattern: "${fileName}",
- sharedPublicly: true
- )
  }
  }
  }