From 445715d7486ade9a25314f1ac82f1d9ef89b9b61 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Fri, 16 Aug 2024 13:59:42 +0300 Subject: [PATCH] updated observability/stream version --- .buildkite/scripts/initCloudEnv.sh | 34 +- .buildkite/x-pack/pipeline.xpack.filebeat.yml | 622 +++++++++--------- testing/environments/docker/cometd/Dockerfile | 2 +- 3 files changed, 313 insertions(+), 345 deletions(-) diff --git a/.buildkite/scripts/initCloudEnv.sh b/.buildkite/scripts/initCloudEnv.sh index 39ce53da917..624c7177a70 100755 --- a/.buildkite/scripts/initCloudEnv.sh +++ b/.buildkite/scripts/initCloudEnv.sh @@ -63,38 +63,6 @@ dockerTeardown() { docker-compose -f .buildkite/deploy/docker/docker-compose.yml down -v } -terraformSetup() { - max_retries=2 - timeout=5 - retries=0 - - while true; do - echo "~~~ Setting up Terraform" - out=$(terraformApply 2>&1) - exit_code=$? - - echo "$out" - - if [ $exit_code -eq 0 ]; then - break - else - retries=$((retries + 1)) - - if [ $retries -gt $max_retries ]; then - teardown - echo "+++ Terraform init & apply failed: $out" - exit 1 - fi - - teardown - - sleep_time=$((timeout * retries)) - echo "~~~~ Retry #$retries failed. Retrying after ${sleep_time}s..." - sleep $sleep_time - fi - done -} - teardown() { terraformDestroy dockerTeardown @@ -104,4 +72,4 @@ trap 'teardown' EXIT exportAwsSecrets dockerUp -terraformSetup +terraformApply diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index df7b39f3f03..7034b4cf641 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -26,273 +26,273 @@ env: TEST_COVERAGE: "true" steps: - - group: "Check/Update" - key: "x-pack-filebeat-check-update" - - steps: - - label: "x-pack/filebeat: Run check/update" - command: | - set -eo pipefail - make -C x-pack/filebeat check update - make check-no-changes - retry: - automatic: - - limit: 3 - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3" - cpu: "4000m" - memory: "8Gi" - useCustomGlobalHooks: true - notify: - - github_commit_status: - context: "x-pack-filebeat: check/update" - - - wait: ~ - # with PRs, we want to run mandatory tests only if check/update step succeed - # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests - # this allows building DRA artifacts even if there is flakiness in check/update step - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: "x-pack-filebeat-check-update" - - - - group: "Mandatory Tests" - key: "x-pack-filebeat-mandatory-tests" - - steps: - - label: ":ubuntu: x-pack/filebeat: Ubuntu x86_64 Unit Tests" - key: "x-pack-filebeat-mandatory-linux-unit-test" - command: | - cd x-pack/filebeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Ubuntu x86_64 Unit Tests" - - - label: ":ubuntu: x-pack/filebeat: Go Integration Tests" - key: "x-pack-filebeat-mandatory-int-test" - command: | - cd x-pack/filebeat - mage goIntegTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Go Integration Tests" - - - label: ":ubuntu: x-pack/filebeat: Python Integration Tests" - key: "x-pack-filebeat-mandatory-python-int-test" - command: | - cd x-pack/filebeat - mage pythonIntegTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Python Integration Tests" - - - label: ":windows: x-pack/filebeat: Win 2022 Unit Tests" - command: | - Set-Location -Path x-pack/filebeat - mage build unitTest - key: "x-pack-filebeat-mandatory-win-2022-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Win 2022 Unit Tests" - - - label: ":windows: x-pack/filebeat: Win 2016 Unit Tests" - command: | - Set-Location -Path x-pack/filebeat - mage build unitTest - key: "x-pack-filebeat-mandatory-win-2016-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Win 2016 Unit Tests" - - - group: "Extended Windows Tests" - key: "x-pack-filebeat-extended-win-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ - - steps: - - label: ":windows: x-pack/filebeat: Win 2019 Unit Tests" - command: | - Set-Location -Path x-pack/filebeat - mage build unitTest - key: "x-pack-filebeat-extended-win-2019-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Win 2019 Unit Tests" - - - label: ":windows: x-pack/filebeat: Win 10 Unit Tests" - command: | - Set-Location -Path x-pack/filebeat - mage build unitTest - key: "x-pack-filebeat-extended-win-10-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Win 10 Unit Tests" - - - label: ":windows: x-pack/filebeat: Win 11 Unit Tests" - command: | - Set-Location -Path x-pack/filebeat - mage build unitTest - key: "x-pack-filebeat-extended-win-11-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Win 11 Unit Tests" +# - group: "Check/Update" +# key: "x-pack-filebeat-check-update" +# +# steps: +# - label: "x-pack/filebeat: Run check/update" +# command: | +# set -eo pipefail +# make -C x-pack/filebeat check update +# make check-no-changes +# retry: +# automatic: +# - limit: 3 +# agents: +# image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3" +# cpu: "4000m" +# memory: "8Gi" +# useCustomGlobalHooks: true +# notify: +# - github_commit_status: +# context: "x-pack-filebeat: check/update" +# +# - wait: ~ +# # with PRs, we want to run mandatory tests only if check/update step succeed +# # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests +# # this allows building DRA artifacts even if there is flakiness in check/update step +# if: build.env("BUILDKITE_PULL_REQUEST") != "false" +# depends_on: "x-pack-filebeat-check-update" +# +# +# - group: "Mandatory Tests" +# key: "x-pack-filebeat-mandatory-tests" +# +# steps: +# - label: ":ubuntu: x-pack/filebeat: Ubuntu x86_64 Unit Tests" +# key: "x-pack-filebeat-mandatory-linux-unit-test" +# command: | +# cd x-pack/filebeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_DEFAULT_MACHINE_TYPE}" +# artifact_paths: +# - "x-pack/filebeat/build/*.xml" +# - "x-pack/filebeat/build/*.json" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: Ubuntu x86_64 Unit Tests" +# +# - label: ":ubuntu: x-pack/filebeat: Go Integration Tests" +# key: "x-pack-filebeat-mandatory-int-test" +# command: | +# cd x-pack/filebeat +# mage goIntegTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_DEFAULT_MACHINE_TYPE}" +# artifact_paths: +# - "x-pack/filebeat/build/*.xml" +# - "x-pack/filebeat/build/*.json" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: Go Integration Tests" +# +# - label: ":ubuntu: x-pack/filebeat: Python Integration Tests" +# key: "x-pack-filebeat-mandatory-python-int-test" +# command: | +# cd x-pack/filebeat +# mage pythonIntegTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_DEFAULT_MACHINE_TYPE}" +# artifact_paths: +# - "x-pack/filebeat/build/*.xml" +# - "x-pack/filebeat/build/*.json" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: Python Integration Tests" +# +# - label: ":windows: x-pack/filebeat: Win 2022 Unit Tests" +# command: | +# Set-Location -Path x-pack/filebeat +# mage build unitTest +# key: "x-pack-filebeat-mandatory-win-2022-unit-tests" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2022}" +# machine_type: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: +# - "x-pack/filebeat/build/*.xml" +# - "x-pack/filebeat/build/*.json" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: Win 2022 Unit Tests" +# +# - label: ":windows: x-pack/filebeat: Win 2016 Unit Tests" +# command: | +# Set-Location -Path x-pack/filebeat +# mage build unitTest +# key: "x-pack-filebeat-mandatory-win-2016-unit-tests" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2016}" +# machine_type: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: +# - "x-pack/filebeat/build/*.xml" +# - "x-pack/filebeat/build/*.json" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: Win 2016 Unit Tests" +# +# - group: "Extended Windows Tests" +# key: "x-pack-filebeat-extended-win-tests" +# if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ +# +# steps: +# - label: ":windows: x-pack/filebeat: Win 2019 Unit Tests" +# command: | +# Set-Location -Path x-pack/filebeat +# mage build unitTest +# key: "x-pack-filebeat-extended-win-2019-unit-tests" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2019}" +# machineType: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: +# - "x-pack/filebeat/build/*.xml" +# - "x-pack/filebeat/build/*.json" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: Win 2019 Unit Tests" +# +# - label: ":windows: x-pack/filebeat: Win 10 Unit Tests" +# command: | +# Set-Location -Path x-pack/filebeat +# mage build unitTest +# key: "x-pack-filebeat-extended-win-10-unit-tests" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_10}" +# machineType: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: +# - "x-pack/filebeat/build/*.xml" +# - "x-pack/filebeat/build/*.json" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: Win 10 Unit Tests" +# +# - label: ":windows: x-pack/filebeat: Win 11 Unit Tests" +# command: | +# Set-Location -Path x-pack/filebeat +# mage build unitTest +# key: "x-pack-filebeat-extended-win-11-unit-tests" +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_11}" +# machineType: "${GCP_WIN_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: +# - "x-pack/filebeat/build/*.xml" +# - "x-pack/filebeat/build/*.json" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: Win 11 Unit Tests" - group: "Extended Tests" key: "x-pack-filebeat-extended-tests" steps: - - label: ":ubuntu: x-pack/filebeat: Ubuntu arm64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - key: "x-pack-filebeat-mandatory-linux-arm-unit-test" - command: | - cd x-pack/filebeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: Ubuntu arm64 Unit Tests" - - - label: ":mac: x-pack/filebeat: macOS x86_64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - cd x-pack/filebeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: macOS x86_64 Unit Tests" - - - label: ":mac: x-pack/filebeat: macOS arm64 Unit Tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - cd x-pack/filebeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: - - "x-pack/filebeat/build/*.xml" - - "x-pack/filebeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/filebeat: macOS arm64 Unit Tests" +# - label: ":ubuntu: x-pack/filebeat: Ubuntu arm64 Unit Tests" +# if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ +# key: "x-pack-filebeat-mandatory-linux-arm-unit-test" +# command: | +# cd x-pack/filebeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "aws" +# imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" +# instanceType: "${AWS_ARM_INSTANCE_TYPE}" +# artifact_paths: +# - "x-pack/filebeat/build/*.xml" +# - "x-pack/filebeat/build/*.json" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: Ubuntu arm64 Unit Tests" +# +# - label: ":mac: x-pack/filebeat: macOS x86_64 Unit Tests" +# if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ +# command: | +# set -euo pipefail +# source .buildkite/scripts/install_macos_tools.sh +# cd x-pack/filebeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_X86_64}" +# artifact_paths: +# - "x-pack/filebeat/build/*.xml" +# - "x-pack/filebeat/build/*.json" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: macOS x86_64 Unit Tests" +# +# - label: ":mac: x-pack/filebeat: macOS arm64 Unit Tests" +# if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ +# command: | +# set -euo pipefail +# source .buildkite/scripts/install_macos_tools.sh +# cd x-pack/filebeat +# mage build unitTest +# retry: +# automatic: +# - limit: 3 +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_ARM}" +# artifact_paths: +# - "x-pack/filebeat/build/*.xml" +# - "x-pack/filebeat/build/*.json" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: macOS arm64 Unit Tests" - label: ":ubuntu: x-pack/filebeat: AWS Tests" key: "x-pack-filebeat-extended-cloud-test" - skip: "skipping as it was on Jenkins: elastic/ingest-dev#3467" + # skip: "skipping as it was on Jenkins: elastic/ingest-dev#3467" # Related issue: https://github.com/elastic/ingest-dev/issues/3467 if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ env: @@ -318,55 +318,55 @@ steps: - github_commit_status: context: "x-pack/filebeat: AWS Tests" - - wait: ~ - # with PRs, we want to run packaging only if mandatory tests succeed - # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests - # this allows building DRA artifacts even if there is flakiness in mandatory tests - if: build.env("BUILDKITE_PULL_REQUEST") != "false" - depends_on: - - step: "x-pack-filebeat-mandatory-tests" - - - group: "Packaging" - key: "x-pack-filebeat-packaging" - - steps: - - label: ":linux: x-pack/filebeat: Packaging Linux" - key: "packaging-linux" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - command: | - cd x-pack/filebeat - mage package - retry: - automatic: - - limit: 3 - timeout_in_minutes: 20 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - notify: - - github_commit_status: - context: "x-pack/filebeat: Packaging Linux" - - - label: ":linux: x-pack/filebeat: Packaging arm64" - key: "packaging-arm" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" - command: | - cd x-pack/filebeat - mage package - retry: - automatic: - - limit: 3 - timeout_in_minutes: 20 - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - notify: - - github_commit_status: - context: "x-pack/filebeat: Packaging Linux arm64" +# - wait: ~ +# # with PRs, we want to run packaging only if mandatory tests succeed +# # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests +# # this allows building DRA artifacts even if there is flakiness in mandatory tests +# if: build.env("BUILDKITE_PULL_REQUEST") != "false" +# depends_on: +# - step: "x-pack-filebeat-mandatory-tests" +# +# - group: "Packaging" +# key: "x-pack-filebeat-packaging" +# +# steps: +# - label: ":linux: x-pack/filebeat: Packaging Linux" +# key: "packaging-linux" +# env: +# PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" +# command: | +# cd x-pack/filebeat +# mage package +# retry: +# automatic: +# - limit: 3 +# timeout_in_minutes: 20 +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# disk_size: 100 +# disk_type: "pd-ssd" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: Packaging Linux" +# +# - label: ":linux: x-pack/filebeat: Packaging arm64" +# key: "packaging-arm" +# env: +# PLATFORMS: "linux/arm64" +# PACKAGES: "docker" +# command: | +# cd x-pack/filebeat +# mage package +# retry: +# automatic: +# - limit: 3 +# timeout_in_minutes: 20 +# agents: +# provider: "aws" +# imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" +# instanceType: "${AWS_ARM_INSTANCE_TYPE}" +# notify: +# - github_commit_status: +# context: "x-pack/filebeat: Packaging Linux arm64" diff --git a/testing/environments/docker/cometd/Dockerfile b/testing/environments/docker/cometd/Dockerfile index 44f19504af2..d001c749e11 100644 --- a/testing/environments/docker/cometd/Dockerfile +++ b/testing/environments/docker/cometd/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/observability/stream:v0.6.1 +FROM docker.elastic.co/observability/stream:v0.7.0 RUN apt update && \ apt -y install curl