feature(sdcm/reporting): Add initial simple elasticsearch reporter

This commit adds a simple elastic reporter for the run status of a
particular job. This acts as a replacement to previously used
db_stats.py, notably being much simpler in both the usage and resulting
documents.

Task: scylladb/qa-tasks#1490
(cherry picked from commit 55ca8fb)

sdcm/reporting/elastic_reporter.py

@@ -0,0 +1,55 @@
+import datetime
+import logging
+
+
+from sdcm.es import ES
+from sdcm.utils.ci_tools import get_job_name, get_job_url
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ElasticRunReporter:
+
+ INDEX_NAME = "sct_test_runs"
+
+ def __init__(self) -> None:
+ self._es = ES()
+
+ @staticmethod
+ def get_build_number(build_job_url: str) -> int | None:
+ build_number = build_job_url.rstrip("/").split("/")[-1] if build_job_url else -1
+ if build_number:
+ try:
+ return int(build_number)
+ except ValueError:
+ LOGGER.error("Error parsing build number from %s: got %s as build_number", build_job_url, build_number)
+ return None
+
+ def report_run(self, run_id: str, status: str, index=None) -> bool:
+ job_name = get_job_name()
+ if job_name == "local_run":
+ LOGGER.warning("Will not report a local run to elastic, aborting.")
+ return False
+ build_url = get_job_url()
+ if not build_url:
+ LOGGER.warning("Build URL is missing, unable to report the run.")
+ return False
+ build_number = self.get_build_number(build_url)
+
+ index = self.INDEX_NAME if not index else index
+ if not self._check_index(index):
+ self._es.indices.create(index=self.INDEX_NAME)
+ document = {
+ "timestamp": datetime.datetime.utcnow(),
+ "run_id": run_id,
+ "status": status,
+ "build_id": job_name,
+ "build_url": build_url,
+ "build_number": build_number,
+ }
+
+ self._es.create(index=index, document=document, id=run_id, doc_type="sct_test_run_short_v1")
+ return True
+
+ def _check_index(self, index_name: str):
+ return self._es.indices.exists(index=index_name)

sdcm/tester.py

@@ -57,6 +57,7 @@
 from sdcm.cluster_k8s.eks import MonitorSetEKS
 from sdcm.provision.azure.provisioner import AzureProvisioner
 from sdcm.provision.provisioner import provisioner_factory
+from sdcm.reporting.elastic_reporter import ElasticRunReporter
 from sdcm.reporting.tooling_reporter import PythonDriverReporter
 from sdcm.scan_operation_thread import ScanOperationThread
 from sdcm.nosql_thread import NoSQLBenchStressThread
@@ -2778,6 +2779,10 @@ def tearDown(self):
 
  self.finalize_teardown()
  self.argus_finalize_test_run()
+ try:
+ ElasticRunReporter().report_run(run_id=self.test_config.test_id(), status=self.get_test_status())
+ except Exception: # pylint: disable=broad-except # noqa: BLE001
+ pass
  self.argus_heartbeat_stop_signal.set()
 
  self.log.info('Test ID: {}'.format(self.test_config.test_id()))