CI: enable AWS cloud testing on a PR basis if changes in certain files

Jenkinsfile

@@ -291,13 +291,13 @@ def generateStages(Map args = [:]) {
 def cloud(Map args = [:]) {
  withGithubNotify(context: args.context) {
  withNode(labels: args.label, forceWorkspace: true){
- startCloudTestEnv(name: args.directory, dirs: args.dirs)
+ startCloudTestEnv(name: args.directory, dirs: args.dirs, withAWS: args.withAWS)
  }
- withCloudTestEnv() {
+ withCloudTestEnv(args) {
  try {
  target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id)
  } finally {
- terraformCleanup(name: args.directory, dir: args.directory)
+ terraformCleanup(name: args.directory, dir: args.directory, withAWS: args.withAWS)
  }
  }
  }
@@ -868,14 +868,15 @@ def tarAndUploadArtifacts(Map args = [:]) {
 * This method executes a closure with credentials for cloud test
 * environments.
 */
-def withCloudTestEnv(Closure body) {
+def withCloudTestEnv(Map args = [:], Closure body) {
  def maskedVars = []
  def testTags = "${env.TEST_TAGS}"
 
  // Allow AWS credentials when the build was configured to do so with:
  // - the cloudtests build parameters
  // - the aws github label
- if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws')) {
+ // - forced with the cloud argument aws github label
+ if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws') || args.get('withAWS', false)) {
  testTags = "${testTags},aws"
  def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data
  if (!aws.containsKey('access_key')) {
@@ -889,6 +890,7 @@ def withCloudTestEnv(Closure body) {
  [var: "AWS_ACCESS_KEY_ID", password: aws.access_key],
  [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key],
  ])
+ log(level: 'INFO', text: 'withCloudTestEnv: it has been configured to run in AWS.')
  }
 
  withEnv([
@@ -914,7 +916,7 @@ def startCloudTestEnv(Map args = [:]) {
  String name = normalise(args.name)
  def dirs = args.get('dirs',[])
  stage("${name}-prepare-cloud-env"){
- withCloudTestEnv() {
+ withCloudTestEnv(args) {
  withBeatsEnv(archive: false, withModule: false) {
  try {
  dirs?.each { folder ->
@@ -957,7 +959,7 @@ def terraformCleanup(Map args = [:]) {
  String name = normalise(args.name)
  String directory = args.dir
  stage("${name}-tear-down-cloud-env"){
- withCloudTestEnv() {
+ withCloudTestEnv(args) {
  withBeatsEnv(archive: false, withModule: false) {
  unstash("terraform-${name}")
  retryWithSleep(retries: 2, seconds: 5, backoff: true) {
@@ -1082,6 +1084,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
  public run(Map args = [:]){
  steps.stageStatusCache(args){
  def withModule = args.content.get('withModule', false)
+ def withAWS = args.content.get('withAWS', false)
  //
  // What's the retry policy for fighting the flakiness:
  // 1) Lint/Packaging/Cloud/k8sTest stages don't retry, since their failures are normally legitim
@@ -1140,7 +1143,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
  steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
  }
  if(args?.content?.containsKey('cloud')) {
- steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id)
+ steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id, withAWS: withAWS)
  }
  }
  }

x-pack/filebeat/Jenkinsfile.yml

@@ -113,6 +113,20 @@ stages:
  labels:
  - "aws"
  stage: extended
+ cloudAWS:
+ cloud: "mage build test"
+ withAWS: true ## Enable the tests to run in AWS
+ withModule: true ## run the ITs only if the changeset affects a specific module.
+ dirs: ## run the cloud tests for the given modules.
+ - "x-pack/filebeat/input/awss3/_meta/terraform"
+ when: ## Override the top-level when.
+ changeset: ## when PR contains any of those entries in the changeset
+ - "^x-pack/filebeat/input/awss3/.*"
+ - "^x-pack/filebeat/module/aws/.*"
+ - "^x-pack/filebeat/input/awscloudwatch/.*"
+ - "^x-pack/filebeat/Jenkinsfile.yml"
+ - "^x-pack/libbeat/common/aws/.*"
+ stage: extended
  #windows-7-32: See https:/elastic/beats/issues/22315
  # mage: "mage build unitTest"
  # platforms: ## override default labels in this specific stage.

x-pack/metricbeat/Jenkinsfile.yml

@@ -45,6 +45,18 @@ stages:
  labels:
  - "aws"
  stage: extended
+ cloudAWS:
+ cloud: "mage build test"
+ withAWS: true ## Enable the tests to run in AWS
+ withModule: true ## run the ITs only if the changeset affects a specific module.
+ dirs: ## run the cloud tests for the given modules.
+ - "x-pack/metricbeat/module/aws"
+ when: ## Override the top-level when.
+ changeset: ## when PR contains any of those entries in the changeset
+ - "^x-pack/metricbeat/module/aws/.*"
+ - "^x-pack/metricbeat/Jenkinsfile.yml"
+ - "^x-pack/libbeat/common/aws/.*"
+ stage: extended
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.