[CI] disable aws cloud testing by default (elastic#23186)

* Disable by default, and added the labels/parameters/comments configuration to be customised on a PR basis if required
* Created a weekly job to run AWS cloud testing for master and 7.x branches

.ci/jobs/beats-schedule-weekly.yml

@@ -0,0 +1,27 @@
+---
+- job:
+ name: beats-schedule-weekly
+ display-name: Jobs scheduled weekly (Sunday)
+ description: Jobs scheduled weekly (Sunday)
+ view: Beats
+ project-type: pipeline
+ parameters:
+ - string:
+ name: branch_specifier
+ default: master
+ description: the Git branch specifier to build
+ pipeline-scm:
+ script-path: .ci/schedule-weekly.groovy
+ scm:
+ - git:
+ url: [email protected]:elastic/beats.git
+ refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+ wipe-workspace: 'True'
+ name: origin
+ shallow-clone: true
+ credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+ reference-repo: /var/lib/jenkins/.git-references/beats.git
+ branches:
+ - $branch_specifier
+ triggers:
+ - timed: 'H H(1-4) * * 0'

.ci/schedule-weekly.groovy

@@ -0,0 +1,33 @@
+@Library('apm@current') _
+
+pipeline {
+ agent none
+ environment {
+ NOTIFY_TO = credentials('notify-to')
+ PIPELINE_LOG_LEVEL = 'INFO'
+ }
+ options {
+ timeout(time: 1, unit: 'HOURS')
+ buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+ timestamps()
+ ansiColor('xterm')
+ disableResume()
+ durabilityHint('PERFORMANCE_OPTIMIZED')
+ }
+ triggers {
+ cron('H H(1-4) * * 0')
+ }
+ stages {
+ stage('Nighly beats builds') {
+ steps {
+ build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)])
+ build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)])
+ }
+ }
+ }
+ post {
+ cleanup {
+ notifyBuildResult(prComment: false)
+ }
+ }
+}

Jenkinsfile

@@ -37,7 +37,7 @@ pipeline {
  }
  parameters {
  booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.')
- booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.')
+ booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.')
  string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
  booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
  booleanParam(name: 'armTest', defaultValue: false, description: 'Allow ARM stages.')

x-pack/metricbeat/Jenkinsfile.yml

@@ -18,11 +18,20 @@ stages:
  make -C x-pack/metricbeat check;
  make -C x-pack/metricbeat update;
  make check-no-changes;
- build:
+ unitTest:
+ mage: "mage build unitTest"
+ cloud:
  cloud: "mage build test"
  withModule: true ## run the ITs only if the changeset affects a specific module.
  dirs: ## run the cloud tests for the given modules.
  - "x-pack/metricbeat/module/aws"
+ when: ## Override the top-level when.
+ parameters:
+ - "awsCloudTests"
+ comments:
+ - "/test x-pack/metricbeat for aws cloud"
+ labels:
+ - "aws"
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.