migrate libbeat pipeline (#37725)

.buildkite/hooks/pre-command

@@ -12,7 +12,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
  fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
  source .buildkite/scripts/setenv.sh
  if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
  echo "Skipped pre-command when running the Upload pipeline"

.buildkite/libbeat/pipeline.libbeat.yml

@@ -1,5 +1,45 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+ IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+ IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+ GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+ GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16"
+ GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+ AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+ BEATS_PROJECT_NAME: "libbeat"
+
 steps:
- - label: "Example test"
- command: echo "Hello!"
+
+ - input: "Input Parameters"
+ key: "input-run-all-stages"
+ fields:
+ - select: "Libbeat - runLibbeat"
+ key: "runLibbeat"
+ options:
+ - label: "True"
+ value: "true"
+ - label: "False"
+ value: "false"
+ default: "false"
+ - select: "Libbeat - runLibBeatArmTest"
+ key: "runLibBeatArmTest"
+ options:
+ - label: "True"
+ value: "true"
+ - label: "False"
+ value: "false"
+ default: "false"
+ if: "build.source == 'ui'"
+
+ - wait: ~
+ if: "build.source == 'ui'"
+ allow_dependency_failure: false
+
+ - label: ":linux: Load dynamic Libbeat pipeline"
+ key: "libbeat-pipeline"
+ command: ".buildkite/scripts/generate_libbeat_pipeline.sh"
+ agents:
+ provider: "gcp"
+ image: "${IMAGE_UBUNTU_X86_64}"
+ machineType: "${GCP_DEFAULT_MACHINE_TYPE}"

.buildkite/metricbeat/pipeline.yml

@@ -10,22 +10,27 @@ env:
  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
  GO_AGENT_IMAGE: "golang:${GO_VERSION}"
  BEATS_PROJECT_NAME: "metricbeat"
+ GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+ GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16"
+ GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+ AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+
 
 steps:
 
  - input: "Input Parameters"
- key: "input-run-all-stages"
+ key: "runMetricbeat"
  fields:
- - select: "Metricbeat - runAllStages"
- key: "runAllStages"
+ - select: "Metricbeat - runMetricbeat"
+ key: "runMetricbeat"
  options:
  - label: "True"
  value: "true"
  - label: "False"
  value: "false"
  default: "false"
- - select: "Metricbeat - runMacOsTests"
- key: "UI_MACOS_TESTS"
+ - select: "Metricbeat - runMetricbeatMacOsTests"
+ key: "runMetricbeatMacOsTests"
  options:
  - label: "True"
  value: "true"
@@ -44,3 +49,4 @@ steps:
  agents:
  provider: "gcp"
  image: "${IMAGE_UBUNTU_X86_64}"
+ machineType: "${GCP_DEFAULT_MACHINE_TYPE}"

.buildkite/scripts/common.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -euo pipefail
 
 WORKSPACE=${WORKSPACE:-"$(pwd)"}
@@ -8,8 +8,11 @@ platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]')
 arch_type="$(uname -m)"
 GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
 ONLY_DOCS=${ONLY_DOCS:-"true"}
-UI_MACOS_TESTS="$(buildkite-agent meta-data get UI_MACOS_TESTS --default ${UI_MACOS_TESTS:-"false"})"
-runAllStages="$(buildkite-agent meta-data get runAllStages --default ${runAllStages:-"false"})"
+runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
+runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})"
+runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
+runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})"
+
 metricbeat_changeset=(
  "^metricbeat/.*"
  "^go.mod"
@@ -175,64 +178,54 @@ are_changed_only_paths() {
  local changed_files=$(git diff --name-only HEAD@{1} HEAD)
  if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then
  return 0
- else
- return 1
  fi
+ return 1
 }
 
 are_conditions_met_mandatory_tests() {
- if [[ "${BUILDKITE_PULL_REQUEST}" == "" ]] || [[ "${runAllStages}" == "true" ]] || [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L107-L137
- if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then # from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
- return 0
- else
- return 1
+ if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+ if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
+ return 0
+ fi
+ elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
+ if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
+ return 0
+ fi
  fi
- else
- return 1
  fi
+ return 1
 }
 
-are_conditions_met_extended_tests() {
+are_conditions_met_libbeat_arm_tests() {
  if are_conditions_met_mandatory_tests; then #from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
- return 0
- else
- return 1
+ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
+ if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runLibBeatArmTest}" == "true" ]]; then
+ return 0
+ fi
+ fi
  fi
+ return 1
 }
 
-are_conditions_met_macos_tests() {
- if are_conditions_met_mandatory_tests; then #from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
- if [[ "${UI_MACOS_TESTS}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
- return 0
- else
- return 1
- fi
- else
- return 1
+are_conditions_met_metricbeat_macos_tests() {
+ if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+ return 0
  fi
+ return 1
 }
 
-are_conditions_met_extended_windows_tests() {
- if [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]] || [[ "${runAllStages}" == "true" ]]; then #from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
- if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then # from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
- return 0
- else
- return 1
- fi
- else
- return 1
+are_conditions_met_packaging() {
+ if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
+ return 0
  fi
+ return 1
 }
 
-are_conditions_met_packaging() {
- if are_conditions_met_extended_windows_tests; then #from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
- if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https:/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
- return 0
- else
- return 1
- fi
- else
- return 1
+config_git() {
+ if [ -z "$(git config --get user.email)" ]; then
+ git config --global user.email "[email protected]"
+ git config --global user.name "beatsmachine"
  fi
 }
 
@@ -249,4 +242,4 @@ fi
 
 if are_paths_changed "${packaging_changeset[@]}" ; then
  PACKAGING_CHANGES="true"
-fi
+fi

.buildkite/scripts/generate_libbeat_pipeline.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+pipelineName="pipeline.libbeat-dynamic.yml"
+
+echo "Add the mandatory and extended tests without additional conditions into the pipeline"
+if are_conditions_met_mandatory_tests; then
+ cat > $pipelineName <<- YAML
+
+steps:
+
+ - group: "Mandatory Tests"
+ key: "mandatory-tests"
+ steps:
+ - label: ":linux: Ubuntu Unit Tests"
+ key: "mandatory-linux-unit-test"
+ command: ".buildkite/scripts/unit_tests.sh"
+ agents:
+ provider: "gcp"
+ image: "${IMAGE_UBUNTU_X86_64}"
+ machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+ artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+ - label: ":go: Go Integration Tests"
+ key: "mandatory-int-test"
+ command: ".buildkite/scripts/go_int_tests.sh"
+ agents:
+ provider: "gcp"
+ image: "${IMAGE_UBUNTU_X86_64}"
+ machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+ artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+ - label: ":python: Python Integration Tests"
+ key: "mandatory-python-int-test"
+ command: ".buildkite/scripts/py_int_tests.sh"
+ agents:
+ provider: "gcp"
+ image: "${IMAGE_UBUNTU_X86_64}"
+ machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+ artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+ - label: ":negative_squared_cross_mark: Cross compile"
+ key: "mandatory-cross-compile"
+ command: ".buildkite/scripts/crosscompile.sh"
+ agents:
+ provider: "gcp"
+ image: "${IMAGE_UBUNTU_X86_64}"
+ machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+ artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml"
+
+ - label: ":testengine: Stress Tests"
+ key: "mandatory-stress-test"
+ command: ".buildkite/scripts/stress_tests.sh"
+ agents:
+ provider: "gcp"
+ image: "${IMAGE_UBUNTU_X86_64}"
+ machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+ artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml"
+
+YAML
+fi
+
+echo "Check and add the Extended Tests into the pipeline"
+if are_conditions_met_libbeat_arm_tests; then
+ cat >> $pipelineName <<- YAML
+
+ - group: "Extended Tests"
+ key: "extended-tests"
+ steps:
+ - label: ":linux: Arm64 Unit Tests"
+ key: "extended-arm64-unit-tests"
+ command: ".buildkite/scripts/unit_tests.sh"
+ agents:
+ provider: "aws"
+ imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+ instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+ artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+YAML
+fi
+
+echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public
+cat $pipelineName
+
+echo "--- Loading dynamic steps"
+buildkite-agent pipeline upload $pipelineName