Merge remote-tracking branch 'upstream/master' into feature/use-debian-9

* upstream/master: (577 commits)
  Remove Journalbeat (elastic#29131)
  Add note that there is no warranty or support for generator code (elastic#28797)
  packetbeat: preparation for npcap addition (elastic#29017)
  Use the generic helper for opening file to read in filestream (elastic#29180)
  Workflow for macos (elastic#29174)
  Fix `decode_json_fields` processor to always add error key (elastic#29107)
  Workflow for macos (elastic#29156)
  Fix agent download timeout values in yaml files (elastic#29039)
  Added workflow file for builds with macos (elastic#29148)
  CI: enable AWS cloud testing on a PR basis if changes in certain files (elastic#29047)
  Remove links to Journalbeat (elastic#29134)
  Fix rds metadata in cloudwatch metricset (elastic#29106)
  [mergify]: notify conflicts in PRs that are still open (elastic#29122)
  Use NamedWatcher in Agent's k8s provider (elastic#29095)
  override host on statsd metricset (elastic#29103)
  Skip config check in autodiscover for duplicated configurations (elastic#29048)
  Change "filebeat.config.modules.enabled" to "true" (elastic#28769)
  Remove deprecated spool queue from Beats (elastic#28869)
  Add `beat` field back to beat.stats (elastic#29094)
  Revert "Move labels and annotations under kubernetes.namespace. (elastic#27917)" (elastic#29069)
  ...

.ci/apm-beats-update.groovy

@@ -126,7 +126,9 @@ def beatsUpdate() {
  git config --global --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*"
 
  go mod edit -replace github.com/elastic/beats/v7=\${GOPATH}/src/github.com/elastic/beats-local
+ go mod tidy
  echo '{"name": "${GOPATH}/src/github.com/elastic/beats-local", "licenceType": "Elastic"}' >> \${GOPATH}/src/github.com/elastic/beats-local/dev-tools/notice/overrides.json
+
  make update
  git commit -a -m beats-update
 

.ci/jobs/beats-schedule-daily.yml

@@ -0,0 +1,27 @@
+---
+- job:
+ name: beats-schedule-daily
+ display-name: Jobs scheduled daily (weekdays)
+ description: Jobs scheduled daily (weekdays)
+ view: Beats
+ project-type: pipeline
+ parameters:
+ - string:
+ name: branch_specifier
+ default: master
+ description: the Git branch specifier to build
+ pipeline-scm:
+ script-path: .ci/schedule-daily.groovy
+ scm:
+ - git:
+ url: [email protected]:elastic/beats.git
+ refspec: +refs/heads/*:refs/remotes/origin/*
+ wipe-workspace: 'True'
+ name: origin
+ shallow-clone: true
+ credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+ reference-repo: /var/lib/jenkins/.git-references/beats.git
+ branches:
+ - $branch_specifier
+ triggers:
+ - timed: 'H H(2-3) * * 1-5'

.ci/jobs/beats-schedule-weekly.yml

@@ -24,4 +24,4 @@
  branches:
  - $branch_specifier
  triggers:
- - timed: 'H H(1-4) * * 0'
+ - timed: 'H H(1-2) * * 0'

.ci/jobs/beats.yml

@@ -17,7 +17,7 @@
  discover-pr-forks-strategy: 'merge-current'
  discover-pr-forks-trust: 'permission'
  discover-pr-origin: 'merge-current'
- head-filter-regex: '(master|6\.[89]|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+ head-filter-regex: '(master|6\.[89]|7\.16|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
  discover-tags: true
  notification-context: "beats-ci"
  repo: 'beats'

.ci/packaging.groovy

@@ -40,8 +40,6 @@ pipeline {
  }
  parameters {
  booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
- booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
- booleanParam(name: 'arm', defaultValue: true, description: 'Allow ARM stages.')
  }
  stages {
  stage('Filter build') {
@@ -103,7 +101,6 @@ pipeline {
  'auditbeat',
  'filebeat',
  'heartbeat',
- 'journalbeat',
  'metricbeat',
  'packetbeat',
  'winlogbeat',
@@ -113,7 +110,6 @@ pipeline {
  'x-pack/filebeat',
  'x-pack/functionbeat',
  'x-pack/heartbeat',
- // 'x-pack/journalbeat',
  'x-pack/metricbeat',
  'x-pack/osquerybeat',
  'x-pack/packetbeat',
@@ -125,12 +121,6 @@ pipeline {
  stage('Package Linux'){
  agent { label 'ubuntu-18 && immutable' }
  options { skipDefaultCheckout() }
- when {
- beforeAgent true
- expression {
- return params.linux
- }
- }
  environment {
  HOME = "${env.WORKSPACE}"
  PLATFORMS = [
@@ -207,7 +197,6 @@ pipeline {
  'auditbeat',
  'filebeat',
  'heartbeat',
- 'journalbeat',
  'metricbeat',
  'packetbeat',
  'x-pack/auditbeat',
@@ -224,12 +213,6 @@ pipeline {
  stage('Package Docker images for linux/arm64'){
  agent { label 'arm' }
  options { skipDefaultCheckout() }
- when {
- beforeAgent true
- expression {
- return params.arm
- }
- }
  environment {
  HOME = "${env.WORKSPACE}"
  PACKAGES = "docker"
@@ -291,8 +274,6 @@ def pushCIDockerImages(Map args = [:]) {
  tagAndPush(beatName: 'filebeat', arch: arch)
  } else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) {
  tagAndPush(beatName: 'heartbeat', arch: arch)
- } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
- tagAndPush(beatName: 'journalbeat', arch: arch)
  } else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) {
  tagAndPush(beatName: 'metricbeat', arch: arch)
  } else if (env?.BEATS_FOLDER?.endsWith('osquerybeat')) {
@@ -335,22 +316,31 @@ def tagAndPush(Map args = [:]) {
  }
  // supported image flavours
  def variants = ["", "-oss", "-ubi8"]
+
+ if(beatName == 'elastic-agent'){
+ variants.add("-complete")
+ variants.add("-cloud")
+ }
+
  variants.each { variant ->
+ // cloud docker images are stored in the private docker namespace.
+ def sourceNamespace = variant.equals('-cloud') ? 'beats-ci' : 'beats'
  tags.each { tag ->
  // TODO:
  // For backward compatibility let's ensure we tag only for amd64, then E2E can benefit from until
  // they support the versioning with the architecture
  if ("${arch}" == "amd64") {
- doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}")
+ doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}", sourceNamespace: sourceNamespace)
  }
- doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}")
+ doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}", sourceNamespace: sourceNamespace)
  }
  }
 }
 
 /**
 * @param beatName name of the Beat
 * @param variant name of the variant used to build the docker image name
+* @param sourceNamespace namespace to be used as source for the docker tag command
 * @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
 * @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
 */
@@ -359,7 +349,8 @@ def doTagAndPush(Map args = [:]) {
  def variant = args.variant
  def sourceTag = args.sourceTag
  def targetTag = args.targetTag
- def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
+ def sourceNamespace = args.sourceNamespace
+ def sourceName = "${DOCKER_REGISTRY}/${sourceNamespace}/${beatName}${variant}:${sourceTag}"
  def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
  def iterations = 0
  retryWithSleep(retries: 3, seconds: 5, backoff: true) {
@@ -420,39 +411,14 @@ def runE2ETests(){
  };
  }
 
- triggerE2ETests(suites)
+ runE2E(runTestsSuites: suites,
+ beatVersion: "${env.BEAT_VERSION}-SNAPSHOT",
+ gitHubCheckName: env.GITHUB_CHECK_E2E_TESTS_NAME,
+ gitHubCheckRepo: env.REPO,
+ gitHubCheckSha1: env.GIT_BASE_COMMIT)
  }
 }
 
-def triggerE2ETests(String suite) {
- echo("Triggering E2E tests for PR-${env.CHANGE_ID}. Test suites: ${suite}.")
-
- def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
- def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
- def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
-
- def parameters = [
- booleanParam(name: 'forceSkipGitChecks', value: true),
- booleanParam(name: 'forceSkipPresubmit', value: true),
- booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
- string(name: 'BEAT_VERSION', value: beatVersion),
- booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
- string(name: 'runTestsSuites', value: suite),
- string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
- string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
- string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
- ]
-
- build(job: "${e2eTestsPipeline}",
- parameters: parameters,
- propagate: false,
- wait: false
- )
-
- def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME}"
- githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
-}
-
 def withMacOSEnv(Closure body){
  withEnvMask( vars: [
  [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],

.ci/schedule-daily.groovy

@@ -0,0 +1,42 @@
+@Library('apm@current') _
+
+pipeline {
+ agent none
+ environment {
+ NOTIFY_TO = credentials('notify-to')
+ PIPELINE_LOG_LEVEL = 'INFO'
+ }
+ options {
+ timeout(time: 1, unit: 'HOURS')
+ buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+ timestamps()
+ ansiColor('xterm')
+ disableResume()
+ durabilityHint('PERFORMANCE_OPTIMIZED')
+ }
+ triggers {
+ cron('H H(2-3) * * 1-5')
+ }
+ stages {
+ stage('Nighly beats builds') {
+ steps {
+ runBuild(quietPeriod: 0, job: 'Beats/beats/master')
+ // This should be `current_8` bump.getCurrentMinorReleaseFor8
+ runBuild(quietPeriod: 2000, job: 'Beats/beats/8.0')
+ // This should be `current_7` bump.getCurrentMinorReleaseFor7 or
+ // `next_minor_7` bump.getNextMinorReleaseFor7
+ runBuild(quietPeriod: 4000, job: 'Beats/beats/7.16')
+ }
+ }
+ }
+ post {
+ cleanup {
+ notifyBuildResult(prComment: false)
+ }
+ }
+}
+
+def runBuild(Map args = [:]) {
+ def jobName = args.job
+ build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'macosTest', value: true)], wait: false, propagate: false)
+}

.ci/schedule-weekly.groovy

@@ -15,13 +15,17 @@ pipeline {
  durabilityHint('PERFORMANCE_OPTIMIZED')
  }
  triggers {
- cron('H H(1-4) * * 0')
+ cron('H H(1-2) * * 0')
  }
  stages {
- stage('Nighly beats builds') {
+ stage('Weekly beats builds') {
  steps {
- build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
- build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+ runBuild(quietPeriod: 0, job: 'Beats/beats/master')
+ // This should be `current_8` bump.getCurrentMinorReleaseFor8
+ runBuild(quietPeriod: 1000, job: 'Beats/beats/8.0')
+ // This should be `current_7` bump.getCurrentMinorReleaseFor7 or
+ // `next_minor_7` bump.getNextMinorReleaseFor7
+ runBuild(quietPeriod: 2000, job: 'Beats/beats/7.16')
  }
  }
  }
@@ -31,3 +35,8 @@ pipeline {
  }
  }
 }
+
+def runBuild(Map args = [:]) {
+ def jobName = args.job
+ build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+}

.ci/scripts/install-go.bat

@@ -0,0 +1,57 @@
+set GOPATH=%WORKSPACE%
+set MAGEFILE_CACHE=%WORKSPACE%\.magefile
+
+set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH%
+
+curl --version >nul 2>&1 && (
+ echo found curl
+) || (
+ choco install curl -y --no-progress --skipdownloadcache
+)
+
+mkdir %WORKSPACE%\bin
+
+IF EXIST "%PROGRAMFILES(X86)%" (
+ REM Force the gvm installation.
+ SET GVM_BIN=gvm.exe
+ curl -L -o %WORKSPACE%\bin\gvm.exe https:/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe
+ IF ERRORLEVEL 1 (
+ REM gvm installation has failed.
+ del bin\gvm.exe /s /f /q
+ exit /b 1
+ )
+) ELSE (
+ REM Windows 7 workers got a broken gvm installation.
+ curl -L -o %WORKSPACE%\bin\gvm.exe https:/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-386.exe
+ IF ERRORLEVEL 1 (
+ REM gvm installation has failed.
+ del bin\gvm.exe /s /f /q
+ exit /b 1
+ )
+)
+
+SET GVM_BIN=gvm.exe
+WHERE /q %GVM_BIN%
+%GVM_BIN% version
+
+REM Install the given go version
+%GVM_BIN% --debug install %GO_VERSION%
+
+REM Configure the given go version
+FOR /f "tokens=*" %%i IN ('"%GVM_BIN%" use %GO_VERSION% --format=batch') DO %%i
+
+go env
+IF ERRORLEVEL 1 (
+ REM go is not configured correctly.
+ rmdir %WORKSPACE%\.gvm /s /q
+ exit /b 1
+)
+
+where mage
+mage -version
+IF ERRORLEVEL 1 (
+ go get github.com/magefile/mage
+ IF ERRORLEVEL 1 (
+ exit /b 1
+ )
+)