From cc8912f6315de802ab7c6aa4eae731db3c32665d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 19 Nov 2021 10:05:22 +0000 Subject: [PATCH 01/10] CI: enable AWS if changes --- Jenkinsfile | 18 +++++++++++------- x-pack/filebeat/Jenkinsfile.yml | 11 +++++++++++ x-pack/metricbeat/Jenkinsfile.yml | 9 +++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 35ae38cb840..0caba9522aa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -291,13 +291,13 @@ def generateStages(Map args = [:]) { def cloud(Map args = [:]) { withGithubNotify(context: args.context) { withNode(labels: args.label, forceWorkspace: true){ - startCloudTestEnv(name: args.directory, dirs: args.dirs) + startCloudTestEnv(name: args.directory, dirs: args.dirs, cloudAWS: args.cloudAWS) } - withCloudTestEnv() { + withCloudTestEnv(args) { try { target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) } finally { - terraformCleanup(name: args.directory, dir: args.directory) + terraformCleanup(name: args.directory, dir: args.directory, cloudAWS: args.cloudAWS) } } } @@ -868,14 +868,15 @@ def tarAndUploadArtifacts(Map args = [:]) { * This method executes a closure with credentials for cloud test * environments. */ -def withCloudTestEnv(Closure body) { +def withCloudTestEnv(Map args = [:], Closure body) { def maskedVars = [] def testTags = "${env.TEST_TAGS}" // Allow AWS credentials when the build was configured to do so with: // - the cloudtests build parameters // - the aws github label - if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws')) { + // - forced with the cloud argument aws github label + if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws') || args.get('cloudAWS', false)) { testTags = "${testTags},aws" def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data if (!aws.containsKey('access_key')) { @@ -914,7 +915,7 @@ def startCloudTestEnv(Map args = [:]) { String name = normalise(args.name) def dirs = args.get('dirs',[]) stage("${name}-prepare-cloud-env"){ - withCloudTestEnv() { + withCloudTestEnv(args) { withBeatsEnv(archive: false, withModule: false) { try { dirs?.each { folder -> @@ -957,7 +958,7 @@ def terraformCleanup(Map args = [:]) { String name = normalise(args.name) String directory = args.dir stage("${name}-tear-down-cloud-env"){ - withCloudTestEnv() { + withCloudTestEnv(args) { withBeatsEnv(archive: false, withModule: false) { unstash("terraform-${name}") retryWithSleep(retries: 2, seconds: 5, backoff: true) { @@ -1142,6 +1143,9 @@ class RunCommand extends co.elastic.beats.BeatsFunction { if(args?.content?.containsKey('cloud')) { steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id) } + if(args?.content?.containsKey('cloudAWS')) { + steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id, cloudAWS: true) + } } } } diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 6cc9fd969f2..6f8c860290e 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -113,6 +113,17 @@ stages: labels: - "aws" stage: extended + cloudAWS: + cloud: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. + dirs: ## run the cloud tests for the given modules. + - "x-pack/filebeat/input/awss3/_meta/terraform" + when: ## Override the top-level when. + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/filebeat/input/awss3/.*" + - "^x-pack/filebeat/module/aws/.*" + - "^x-pack/filebeat/input/awscloudwatch/.*" + stage: extended #windows-7-32: See https://github.com/elastic/beats/issues/22315 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index a9be741ddae..ce305328d05 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -45,6 +45,15 @@ stages: labels: - "aws" stage: extended + cloudAWS: + cloud: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. + dirs: ## run the cloud tests for the given modules. + - "x-pack/metricbeat/module/aws" + when: ## Override the top-level when. + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/metricbeat/module/aws/.*" + stage: extended macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From 8d60b8f05eab2b9345d5ecb7917153d0db541663 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 19 Nov 2021 11:54:18 +0000 Subject: [PATCH 02/10] Test build --- x-pack/filebeat/module/aws/test.build | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 x-pack/filebeat/module/aws/test.build diff --git a/x-pack/filebeat/module/aws/test.build b/x-pack/filebeat/module/aws/test.build new file mode 100644 index 00000000000..e69de29bb2d From 2e882e8c014b5f41bcf04f2bce0a8c2b478c0f0d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 22 Nov 2021 17:44:40 +0000 Subject: [PATCH 03/10] chore: for testing purposes --- x-pack/metricbeat/module/aws/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 x-pack/metricbeat/module/aws/test.txt diff --git a/x-pack/metricbeat/module/aws/test.txt b/x-pack/metricbeat/module/aws/test.txt new file mode 100644 index 00000000000..e69de29bb2d From f3052798a168821fefe32c4cc24387c4ddcb6f91 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 22 Nov 2021 17:47:48 +0000 Subject: [PATCH 04/10] add logs --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 0caba9522aa..a3c3f97b3ab 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -890,6 +890,7 @@ def withCloudTestEnv(Map args = [:], Closure body) { [var: "AWS_ACCESS_KEY_ID", password: aws.access_key], [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key], ]) + log(level: 'INFO', text: 'withCloudTestEnv: it has been configured to run in AWS.') } withEnv([ From 455013454d69df1eb84a6edea14c1ec27f796168 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 23 Nov 2021 11:37:19 +0000 Subject: [PATCH 05/10] Run as part of the mandatory stage There are genuine test failures for the integ tests therefore the `extended` stage cannot be executed. --- x-pack/filebeat/Jenkinsfile.yml | 2 +- x-pack/metricbeat/Jenkinsfile.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 6f8c860290e..ee80411e1b8 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -123,7 +123,7 @@ stages: - "^x-pack/filebeat/input/awss3/.*" - "^x-pack/filebeat/module/aws/.*" - "^x-pack/filebeat/input/awscloudwatch/.*" - stage: extended + stage: mandatory #windows-7-32: See https://github.com/elastic/beats/issues/22315 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index ce305328d05..24a02844a18 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -53,7 +53,7 @@ stages: when: ## Override the top-level when. changeset: ## when PR contains any of those entries in the changeset - "^x-pack/metricbeat/module/aws/.*" - stage: extended + stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From 1c7c731d8ab604447c4bc36fd975cf2b276f6687 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 24 Nov 2021 09:24:55 +0000 Subject: [PATCH 06/10] Enable the right stage and listen for changes in the pipeline too --- x-pack/filebeat/Jenkinsfile.yml | 3 ++- x-pack/metricbeat/Jenkinsfile.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index ee80411e1b8..89b9312feea 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -114,7 +114,7 @@ stages: - "aws" stage: extended cloudAWS: - cloud: "mage build test" + cloudAWS: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. dirs: ## run the cloud tests for the given modules. - "x-pack/filebeat/input/awss3/_meta/terraform" @@ -123,6 +123,7 @@ stages: - "^x-pack/filebeat/input/awss3/.*" - "^x-pack/filebeat/module/aws/.*" - "^x-pack/filebeat/input/awscloudwatch/.*" + - "^x-pack/filebeat/Jenkinsfile.yml" stage: mandatory #windows-7-32: See https://github.com/elastic/beats/issues/22315 # mage: "mage build unitTest" diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 24a02844a18..4fc2290ee4a 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -46,13 +46,14 @@ stages: - "aws" stage: extended cloudAWS: - cloud: "mage build test" + cloudAWS: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. dirs: ## run the cloud tests for the given modules. - "x-pack/metricbeat/module/aws" when: ## Override the top-level when. changeset: ## when PR contains any of those entries in the changeset - "^x-pack/metricbeat/module/aws/.*" + - "^x-pack/metricbeat/Jenkinsfile.yml" stage: mandatory macos: mage: "mage build unitTest" From c08a7c3b3491f70dba50a07f9895b10e5ba37687 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 24 Nov 2021 09:25:18 +0000 Subject: [PATCH 07/10] remove test --- x-pack/filebeat/module/aws/test.build | 0 x-pack/metricbeat/module/aws/test.txt | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 x-pack/filebeat/module/aws/test.build delete mode 100644 x-pack/metricbeat/module/aws/test.txt diff --git a/x-pack/filebeat/module/aws/test.build b/x-pack/filebeat/module/aws/test.build deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/x-pack/metricbeat/module/aws/test.txt b/x-pack/metricbeat/module/aws/test.txt deleted file mode 100644 index e69de29bb2d..00000000000 From 42e182184f5a79bf027071d01ddc059cee0d0323 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 24 Nov 2021 11:57:50 +0000 Subject: [PATCH 08/10] Use withAWS option to reuse the cloud function --- Jenkinsfile | 12 +++++------- x-pack/filebeat/Jenkinsfile.yml | 3 ++- x-pack/metricbeat/Jenkinsfile.yml | 3 ++- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a3c3f97b3ab..9d52f95553f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -291,13 +291,13 @@ def generateStages(Map args = [:]) { def cloud(Map args = [:]) { withGithubNotify(context: args.context) { withNode(labels: args.label, forceWorkspace: true){ - startCloudTestEnv(name: args.directory, dirs: args.dirs, cloudAWS: args.cloudAWS) + startCloudTestEnv(name: args.directory, dirs: args.dirs, withAWS: args.withAWS) } withCloudTestEnv(args) { try { target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) } finally { - terraformCleanup(name: args.directory, dir: args.directory, cloudAWS: args.cloudAWS) + terraformCleanup(name: args.directory, dir: args.directory, withAWS: args.withAWS) } } } @@ -876,7 +876,7 @@ def withCloudTestEnv(Map args = [:], Closure body) { // - the cloudtests build parameters // - the aws github label // - forced with the cloud argument aws github label - if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws') || args.get('cloudAWS', false)) { + if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws') || args.get('withAWS', false)) { testTags = "${testTags},aws" def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data if (!aws.containsKey('access_key')) { @@ -1084,6 +1084,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction { public run(Map args = [:]){ steps.stageStatusCache(args){ def withModule = args.content.get('withModule', false) + def withAWS = args.content.get('withAWS', false) // // What's the retry policy for fighting the flakiness: // 1) Lint/Packaging/Cloud/k8sTest stages don't retry, since their failures are normally legitim @@ -1142,10 +1143,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction { steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) } if(args?.content?.containsKey('cloud')) { - steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id) - } - if(args?.content?.containsKey('cloudAWS')) { - steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id, cloudAWS: true) + steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id, withAWS: withAWS) } } } diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 89b9312feea..3784faeb7bf 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -114,7 +114,8 @@ stages: - "aws" stage: extended cloudAWS: - cloudAWS: "mage build test" + cloud: "mage build test" + withAWS: true ## Enable the tests to run in AWS withModule: true ## run the ITs only if the changeset affects a specific module. dirs: ## run the cloud tests for the given modules. - "x-pack/filebeat/input/awss3/_meta/terraform" diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 4fc2290ee4a..ba6baae1efb 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -46,7 +46,8 @@ stages: - "aws" stage: extended cloudAWS: - cloudAWS: "mage build test" + cloud: "mage build test" + withAWS: true ## Enable the tests to run in AWS withModule: true ## run the ITs only if the changeset affects a specific module. dirs: ## run the cloud tests for the given modules. - "x-pack/metricbeat/module/aws" From 8d6b0a65176f6ade4935b90fa4adada1946a8c65 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 24 Nov 2021 11:59:16 +0000 Subject: [PATCH 09/10] Add changeset x-pack/libbeat/common/aws/.* --- x-pack/filebeat/Jenkinsfile.yml | 1 + x-pack/metricbeat/Jenkinsfile.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 3784faeb7bf..8d829c35449 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -125,6 +125,7 @@ stages: - "^x-pack/filebeat/module/aws/.*" - "^x-pack/filebeat/input/awscloudwatch/.*" - "^x-pack/filebeat/Jenkinsfile.yml" + - "^x-pack/libbeat/common/aws/.*" stage: mandatory #windows-7-32: See https://github.com/elastic/beats/issues/22315 # mage: "mage build unitTest" diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index ba6baae1efb..87b4d54c5d5 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -55,6 +55,7 @@ stages: changeset: ## when PR contains any of those entries in the changeset - "^x-pack/metricbeat/module/aws/.*" - "^x-pack/metricbeat/Jenkinsfile.yml" + - "^x-pack/libbeat/common/aws/.*" stage: mandatory macos: mage: "mage build unitTest" From 98c5f4143b89ae2804a05a0c008e8feae72dcda0 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 24 Nov 2021 17:39:41 +0000 Subject: [PATCH 10/10] Apply suggestions from code review --- x-pack/filebeat/Jenkinsfile.yml | 2 +- x-pack/metricbeat/Jenkinsfile.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 8d829c35449..8ea1c14fa03 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -126,7 +126,7 @@ stages: - "^x-pack/filebeat/input/awscloudwatch/.*" - "^x-pack/filebeat/Jenkinsfile.yml" - "^x-pack/libbeat/common/aws/.*" - stage: mandatory + stage: extended #windows-7-32: See https://github.com/elastic/beats/issues/22315 # mage: "mage build unitTest" # platforms: ## override default labels in this specific stage. diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 87b4d54c5d5..7a619ed6f16 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -56,7 +56,7 @@ stages: - "^x-pack/metricbeat/module/aws/.*" - "^x-pack/metricbeat/Jenkinsfile.yml" - "^x-pack/libbeat/common/aws/.*" - stage: mandatory + stage: extended macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage.