From 366085dc633ba6a1766ab80bda44a7a77a6be0e5 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Thu, 2 May 2024 16:31:48 +0300 Subject: [PATCH] Remove python dynamic pipeline scripts (#39187) Remove python dynamic pipeline scripts. Signed-off-by: Alexandros Sapranidis --- .buildkite/buildkite.yml | 44 -- .buildkite/pipeline.py | 433 ------------------ .buildkite/pytest.ini | 11 - .../scripts/run_dynamic_pipeline_tests.sh | 43 -- .buildkite/test_pipeline.py | 71 --- auditbeat/buildkite.yml | 56 --- filebeat/buildkite.yml | 60 --- 7 files changed, 718 deletions(-) delete mode 100644 .buildkite/buildkite.yml delete mode 100755 .buildkite/pipeline.py delete mode 100644 .buildkite/pytest.ini delete mode 100755 .buildkite/scripts/run_dynamic_pipeline_tests.sh delete mode 100644 .buildkite/test_pipeline.py delete mode 100644 auditbeat/buildkite.yml delete mode 100644 filebeat/buildkite.yml diff --git a/.buildkite/buildkite.yml b/.buildkite/buildkite.yml deleted file mode 100644 index 4707707e07c..00000000000 --- a/.buildkite/buildkite.yml +++ /dev/null @@ -1,44 +0,0 @@ -projects: - - "auditbeat" - - "deploy/kubernetes" - - "filebeat" - - "heartbeat" - - "libbeat" - - "metricbeat" - - "packetbeat" - - "winlogbeat" - - "x-pack/auditbeat" - - "x-pack/dockerlogbeat" - - "x-pack/filebeat" - - "x-pack/functionbeat" - - "x-pack/heartbeat" - - "x-pack/libbeat" - - "x-pack/metricbeat" - - "x-pack/osquerybeat" - - "x-pack/packetbeat" - - "x-pack/winlogbeat" - -## Changeset macros that are defined here and used in each specific 3.0 pipeline. -changeset: - ci: - - "^Jenkinsfile" - - "^\\.ci/scripts/.*" - oss: - - "^go.mod" - - "^pytest.ini" - - "^dev-tools/.*" - - "^libbeat/.*" - - "^testing/.*" - xpack: - - "^go.mod" - - "^pytest.ini" - - "^dev-tools/.*" - - "^libbeat/.*" - - "^testing/.*" - - "^x-pack/libbeat/.*" - -disabled: - when: - labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. - - skip-ci - draft: true ## Skip the GitHub Pull Request builds with Draft PRs. diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py deleted file mode 100755 index 0207b24bca7..00000000000 --- a/.buildkite/pipeline.py +++ /dev/null @@ -1,433 +0,0 @@ -#!/usr/bin/env python3 -import fnmatch -import os -import subprocess -import sys -from typing import Any - -from ruamel.yaml import YAML - - -class Agent: - """Buildkite Agent object""" - - def __init__(self, image: str, provider: str): - self.image: str = image - self.provider: str = provider - - def create_entity(self): - raise NotImplementedError("Not implemented yet") - - -class AWSAgent(Agent): - """AWS Agent object""" - - def __init__(self, image: str, instance_type: str = None): - super().__init__(image, "aws") - if instance_type is None: - self.instance_type: str = "t4g.large" - else: - self.instance_type = instance_type - - def create_entity(self) -> dict[str, str]: - return { - "provider": self.provider, - "imagePrefix": self.image, - "instanceType": self.instance_type, - } - - -class GCPAgent(Agent): - """GCP Agent object""" - - def __init__(self, image: str): - super().__init__(image, "gcp") - - def create_entity(self) -> dict[str, str]: - return { - "provider": self.provider, - "image": self.image, - } - - -class OrkaAgent(Agent): - """Orka Agent object""" - - def __init__(self, image: str): - super().__init__(image, "orka") - - def create_entity(self) -> dict[str, str]: - return { - "provider": self.provider, - "imagePrefix": self.image, - } - - -class Step: - """Buildkite Step object""" - - def __init__( - self, - name: str, - project: str, - category: str, - agent: Agent, - definition: dict[str, Any], - ): - self.command = definition.get("command", "") - self.env = definition.get("env", {}) - self.agent: Agent = agent - self.name: str = name - self.project: str = project - self.category: str = category - self.comment = "/test " + self.project + " " + self.name - self.label = self.name - - def __lt__(self, other): - return self.name < other.name - - def step_command(self) -> list[str]: - commands = [ - f"cd {self.project}", - self.command, - ] - return commands - - def create_entity(self) -> dict[str, Any]: - data = { - "label": f"{self.project} {self.name}", - "command": self.step_command(), - "notify": [ - { - "github_commit_status": { - "context": f"{self.project.title()}: {self.name}", - } - } - ], - "agents": self.agent.create_entity(), - "artifact_paths": [ - f"{self.project}/build/*.xml", - f"{self.project}/build/*.json", - ], - } - if self.env: - data["env"] = self.env - return data - - -class Group: - """Buildkite Group object""" - - def __init__(self, project: str, category: str, steps: list[Step]): - self.project: str = project - self.category: str = category - self.steps: list[Step] = steps - - def __lt__(self, other): - return self.project < other.project - - def create_entity(self) -> dict[str, Any]: - if len(self.steps) == 0: - return {} - - data = { - "group": f"{self.project} {self.category}", - "key": f"{self.project}-{self.category}", - "steps": [step.create_entity() for step in self.steps], - } - - return data - - -class GitHelper: - def __init__(self): - self.files: list[str] = [] - - def get_pr_changeset(self) -> list[str]: - hash = ["git", "rev-parse", "8.13"] - diff_command = ["git", "diff", "--name-only", "{}...HEAD".format(hash)] - result = subprocess.run(diff_command, stdout=subprocess.PIPE) - - if result.returncode == 0: - self.files = result.stdout.decode().splitlines() - else: - print(f"Detecting changed files failed, exiting [{result.returncode}]") - exit(result.returncode) - return self.files - - -class BuildkitePipeline: - """Buildkite Pipeline object""" - - def __init__(self, groups: list[Group] = None): - if groups is None: - groups = [] - self.groups: list[Group] = groups - - def create_entity(self): - data = {"steps": [group.create_entity() for group in self.groups]} - return data - - -def is_pr() -> bool: - return os.getenv("BUILDKITE_PULL_REQUEST") != "false" - - -def group_comment(group: Group) -> bool: - comment = os.getenv("GITHUB_PR_TRIGGER_COMMENT") - if comment: - # the comment should be a subset of the values - # in .buildkite/pull-requests.json - # TODO: change /test - comment_prefix = "buildkite test" - if group.category == "mandatory": - # i.e: /test filebeat - return comment_prefix + " " + group.project in comment - else: - # i.e: test filebeat extended - return ( - comment_prefix + " " + group.project + " " + group.category in comment - ) - - -def filter_files_by_glob(files, patterns: list[str]): - for pattern in patterns: - # TODO: Support glob extended patterns: ^ and etc. - # Now it supports only linux glob syntax - if fnmatch.filter(files, pattern): - return True - return False - - -def is_in_pr_changeset( - project_changeset_filters: list[str], changeset: list[str] -) -> bool: - return filter_files_by_glob(changeset, project_changeset_filters) - - -def is_group_enabled( - group: Group, changeset_filters: list[str], changeset: list[str] -) -> bool: - if not is_pr(): - return True - - if ( - is_pr() - and is_in_pr_changeset(changeset_filters, changeset) - and group.category.startswith("mandatory") - ): - return True - - return group_comment(group) - - -def fetch_stage(name: str, stage, project: str, category: str) -> Step: - """Create a step given the yaml object.""" - - agent: Agent = None - if ("provider" not in stage) or stage["provider"] == "gcp": - agent = GCPAgent(image=stage["platform"]) - elif stage["provider"] == "aws": - agent = AWSAgent( - image=stage["platform"], - ) - elif stage["provider"] == "orka": - agent = OrkaAgent(image=stage["platform"]) - - return Step( - category=category, name=name, agent=agent, project=project, definition=stage - ) - - -def fetch_group(stages, project: str, category: str) -> Group: - """Create a group given the yaml object.""" - - steps = [] - - for stage in stages: - steps.append( - fetch_stage( - category=category, name=stage, project=project, stage=stages[stage] - ) - ) - - return Group(project=project, category=category, steps=steps) - - -def fetch_pr_pipeline(yaml: YAML) -> list[Group]: - git_helper = GitHelper() - changeset = git_helper.get_pr_changeset() - groups: list[Group] = [] - doc = pipeline_loader(yaml) - for project in doc["projects"]: - project_file = os.path.join(project, "buildkite.yml") - if not os.path.isfile(project_file): - continue - project_obj = project_loader(yaml, project_file) - group = fetch_group( - stages=project_obj["stages"]["mandatory"], - project=project, - category="mandatory", - ) - - if is_group_enabled(group, project_obj["when"]["changeset"], changeset): - groups.append(group) - - group = fetch_group( - stages=project_obj["stages"]["extended"], - project=project, - category="extended", - ) - - if is_group_enabled(group, project_obj["when"]["changeset"], changeset): - groups.append(group) - - # TODO: improve this merging lists - all_groups = [] - for group in groups: - all_groups.append(group) - - return all_groups - - -class PRComment: - command: str - group: str - project: str - step: str - - def __init__(self, comment: str): - words = comment.split() - self.command = words.pop(0) if words else "" - self.project = words.pop(0) if words else "" - self.group = words.pop(0) if words else "" - self.step = words.pop(0) if words else "" - - -# A comment like "/test filebeat extended" -# Returns a group of steps corresponding to the comment -def fetch_pr_comment_group_pipeline(comment: PRComment, yaml: YAML) -> list[Group]: - groups = [] - doc = pipeline_loader(yaml) - if comment.project in doc["projects"]: - project_file = os.path.join(comment.project, "buildkite.yml") - if not os.path.isfile(project_file): - raise FileNotFoundError( - "buildkite.yml not found in: " + "{}".format(comment.project) - ) - project_obj = project_loader(yaml, project_file) - if not project_obj["stages"][comment.group]: - raise ValueError( - "Group not found in {} buildkite.yml: {}".format( - comment.project, comment.group - ) - ) - - group = fetch_group( - stages=project_obj["stages"][comment.group], - project=comment.project, - category="mandatory", - ) - groups.append(group) - - return groups - - -# A comment like "/test filebeat extended unitTest-macos" -def fetch_pr_comment_step_pipeline(comment: PRComment, yaml: YAML) -> list[Group]: - groups = [] - doc = pipeline_loader(yaml) - if comment.project in doc["projects"]: - project_file = os.path.join(comment.project, "buildkite.yml") - if not os.path.isfile(project_file): - raise FileNotFoundError( - "buildkite.yml not found in: " + "{}".format(comment.project) - ) - project_obj = project_loader(yaml, project_file) - if not project_obj["stages"][comment.group]: - raise ValueError( - "Group not found in {} buildkite.yml: {}".format( - comment.project, comment.group - ) - ) - group = fetch_group( - stages=project_obj["stages"][comment.group], - project=comment.project, - category="mandatory", - ) - - filtered_steps = list( - filter(lambda step: step.name == comment.step, group.steps) - ) - - if not filtered_steps: - raise ValueError( - "Step {} not found in {} buildkite.yml".format( - comment.step, comment.project - ) - ) - group.steps = filtered_steps - groups.append(group) - - return groups - - -def pr_comment_pipeline(pr_comment: PRComment, yaml: YAML) -> list[Group]: - - if pr_comment.command == "/test": - - # A comment like "/test" for a PR - # We rerun the PR pipeline - if not pr_comment.group: - return fetch_pr_pipeline(yaml) - - # A comment like "/test filebeat" - # We don't know what group to run hence raise an error - if pr_comment.project and not pr_comment.group: - raise ValueError( - "Specify group or/and step for {}".format(pr_comment.project) - ) - - # A comment like "/test filebeat extended" - # We rerun the filebeat extended pipeline for the PR - if pr_comment.group and not pr_comment.step: - return fetch_pr_comment_group_pipeline(pr_comment, yaml) - - # A comment like "/test filebeat extended unitTest-macos" - if pr_comment.step: - return fetch_pr_comment_step_pipeline(pr_comment, yaml) - - -# TODO: validate unique stages! -def main() -> None: - yaml = YAML(typ="safe") - all_groups = [] - if is_pr(): - if os.getenv("GITHUB_PR_TRIGGER_COMMENT"): - comment = PRComment(os.getenv("GITHUB_PR_TRIGGER_COMMENT")) - all_groups = pr_comment_pipeline(comment, yaml) - else: - all_groups = fetch_pr_pipeline(yaml) - # TODO what to load when not in PR - - # Produce the dynamic pipeline - print( - "# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json" - ) - yaml.dump(BuildkitePipeline(all_groups).create_entity(), sys.stdout) - - -def pipeline_loader(yaml: YAML = YAML(typ="safe")): - with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: - return yaml.load(file) - - -def project_loader(yaml: YAML = YAML(typ="safe"), project_file: str = ""): - with open(project_file, "r", encoding="utf8") as project_fp: - return yaml.load(project_fp) - - -if __name__ == "__main__": - - # pylint: disable=E1120 - main() diff --git a/.buildkite/pytest.ini b/.buildkite/pytest.ini deleted file mode 100644 index 3eff7473d9f..00000000000 --- a/.buildkite/pytest.ini +++ /dev/null @@ -1,11 +0,0 @@ -[pytest] -junit_family=xunit1 - -addopts = --strict-markers -markers = - load: Load tests - tag(name): Tag tests with Go-like semantics - -# Ignore setup and teardown for the timeout -#timeout_func_only = True - diff --git a/.buildkite/scripts/run_dynamic_pipeline_tests.sh b/.buildkite/scripts/run_dynamic_pipeline_tests.sh deleted file mode 100755 index 8eb72d7a96b..00000000000 --- a/.buildkite/scripts/run_dynamic_pipeline_tests.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Run tests for the dynamic pipeline generator only if it's a PR and related files have been changed -# this will allow us to fail fast, if e.g. a PR has broken the generator - -set -euo pipefail - -are_paths_changed() { - local patterns=("${@}") - local changelist=() - for pattern in "${patterns[@]}"; do - changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) - if [ "${#changed_files[@]}" -gt 0 ]; then - changelist+=("${changed_files[@]}") - fi - done - - if [ "${#changelist[@]}" -gt 0 ]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - echo "No files changed within specified changeset:" - echo "${patterns[*]}" - return 1 - fi -} - -pipeline_generator_changeset=( - "^.buildkite/pipeline.py" - "^*/buildkite.yml" -) - -if ! are_paths_changed "${pipeline_generator_changeset[@]}" || [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then - echo "~~~ Skipping pipeline generator tests" - exit -fi - -echo "~~~ Running pipeline generator tests" - -python3 -mpip install --quiet "pytest" -pushd .buildkite -pytest . -popd diff --git a/.buildkite/test_pipeline.py b/.buildkite/test_pipeline.py deleted file mode 100644 index 75fd949ccc8..00000000000 --- a/.buildkite/test_pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -import os - -import pytest -import pipeline - - -@pytest.fixture -def ubuntu2204_aws_agent(): - return { - "command": "fake-cmd", - "platform": "platform-ingest-beats-ubuntu-2204-aarch64", - "provider": "aws" - } - - -@pytest.fixture() -def fake_simple_group(): - return { - "unitTest": { - "command": "fake-cmd", - "platform": "family/platform-ingest-beats-ubuntu-2204", - }, - "integrationTest": { - "command": "fake-integration", - "platform": "family/platform-ingest-beats-ubuntu-2204", - "env": { - "FOO": "BAR", - }, - }, - } - - -def test_fetch_stage(ubuntu2204_aws_agent): - step = pipeline.fetch_stage("test", ubuntu2204_aws_agent, "fake", "fake-category") - assert step.create_entity() == { - "label": "fake test", - "command": ["cd fake", "fake-cmd"], - "notify": [ - { - "github_commit_status": { - "context": "Fake: test", - } - } - ], - "agents": { - "provider": "aws", - "imagePrefix": "platform-ingest-beats-ubuntu-2204-aarch64", - "instanceType": "t4g.large", - }, - "artifact_paths": [ - "fake/build/*.xml", - "fake/build/*.json", - ], - } - - -def test_fetch_group(fake_simple_group): - group = pipeline.fetch_group(fake_simple_group, "fake-project", "testing") - assert len(group.steps) == 2 - for step in group.steps: - assert "testing" == step.category - assert "gcp" == step.agent.provider - - assert group.steps[1].env.get("FOO") == "BAR" - - -def test_is_pr(): - os.environ["BUILDKITE_PULL_REQUEST"] = "1234" - assert pipeline.is_pr() is True - os.environ["BUILDKITE_PULL_REQUEST"] = "false" - assert pipeline.is_pr() is False diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml deleted file mode 100644 index 2abf9d68407..00000000000 --- a/auditbeat/buildkite.yml +++ /dev/null @@ -1,56 +0,0 @@ -when: - changeset: ## when PR contains any of those entries in the changeset - - "auditbeat/**" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss -stages: - # mandatory stage - it runs always for: - # - branches/tags - # - on PRs - # - GitHub comment /test auditbeat - # - GitHub label auditbeat - mandatory: - # NOTE: stage name should be unique! - unitTest: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - crosscompile: - command: "make crosscompile" - platform: "family/platform-ingest-beats-ubuntu-2204" - env: - GOX_FLAGS: "-arch amd64" - unitTest-rhel-9: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-rhel-9" - unitTest-windows-2022: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2022" - unitTest-windows-2016: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2016" - # optional stage - it runs on: - # - branches/tags - # - on PRs if: - # - GitHub comment /test auditbeat . i.e: /test auditbeat integTest - # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... - extended: - # NOTE: stage name should be unique! - integTest: - command: "mage build integTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - integTest-arm: - command: "mage build integTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - unitTest-arm: - command: "mage build unitTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" - unitTest-macos: - command: "mage build unitTest" - platform: "generic-13-ventura-x64" - provider: "orka" - unitTest-windows-2019: - command: "mage build unitTest" - platform: "family/core-windows-2019" diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml deleted file mode 100644 index 3fcabc5f1ce..00000000000 --- a/filebeat/buildkite.yml +++ /dev/null @@ -1,60 +0,0 @@ -when: - branches: true ## for all the branches - changeset: ## when PR contains any of those entries in the changeset - - "filebeat/**" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss -stages: - mandatory: - unitTest: - command: "mage unitTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - goIntegTest: - command: "mage goIntegTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - pythonIntegTest: - command: "mage pythonIntegTest" - platform: "family/platform-ingest-beats-ubuntu-2204" - unitTest-windows-2022: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2022" - unitTest-windows-2016: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2016" - - extended_win: - unitTest-windows-2019: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-2019" - unitTest-windows-11: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-11" - unitTest-windows-10: - command: "mage build unitTest" - platform: "family/platform-ingest-beats-windows-10" - extended: - unitTest-arm: - command: "mage build unitTest" - platform: "platform-ingest-beats-ubuntu-2204-aarch64" - provider: "aws" # move this inside the platform leaf - when: - comments: - - "/test filebeat for arm" - labels: - - "arm" - parameters: - - "armTest" - branches: true ## for all the branches - tags: true ## for all the tags - unitTest-macos: - command: ".buildkite/filebeat/scripts/unit-tests.sh" - platform: "generic-13-ventura-x64" - provider: "orka" - when: - comments: - - "/test filebeat for macos" - labels: - - "macOS" - parameters: - - "macosTest" - tags: true ## for all the tags