From 268d21c1b710d9a80953cdec6ff90ab51e562f24 Mon Sep 17 00:00:00 2001 From: Natalia Bubakova Date: Mon, 4 Sep 2023 18:29:28 +0200 Subject: [PATCH 1/7] Extension of the `ReportPortal` plugin using API [WIP] First part - grouping plans into one launch * option `--merge` to create suite per plan and merge them all in one launch (stored launch uuid in run of the first plan) * option '--attributes' for additional attributes, but mainly for assignment of attributes to the launch with merged plans * option `--uuid` to append new plans to an existing launch * store launch uuid as rp_uuid per merged run, and as launch_uuid per each plan, store launch_url per each plan * rewritten environment variables to the uniform form TMT_PLUGIN_REPORT_REPORTPORTAL_${option} note: skipped mypy errors unresolved --- spec/plans/report.fmf | 12 +- tmt/base.py | 5 +- tmt/schemas/report/reportportal.yaml | 5 + tmt/steps/report/reportportal.py | 204 +++++++++++++++++++-------- 4 files changed, 161 insertions(+), 65 deletions(-) diff --git a/spec/plans/report.fmf b/spec/plans/report.fmf index a26c46ca23..ba9b78b9ca 100644 --- a/spec/plans/report.fmf +++ b/spec/plans/report.fmf @@ -105,18 +105,20 @@ description: to detailed test output and other test information. description: Fill json with test results and other fmf data per each plan, - and send it to a Report Portal instance via its API. + and send it to a Report Portal instance via its API + with token, url and project name given. example: - | - # Set environment variables with the server url and token - export TMT_REPORT_REPORTPORTAL_URL= - export TMT_REPORT_REPORTPORTAL_TOKEN= + # Set environment variables accoriding to TMT_PLUGIN_REPORT_REPORTPORTAL_${OPTION} + export TMT_PLUGIN_REPORT_REPORTPORTAL_URL=${url-to-RP-instance} + export TMT_PLUGIN_REPORT_REPORTPORTAL_TOKEN=${token-from-RP-profile} - | # Enable ReportPortal report from the command line tmt run --all report --how reportportal --project=baseosqe - tmt run --all report --how reportportal --project=baseosqe --exclude-variables="^(TMT|PACKIT|TESTING_FARM).*" tmt run --all report --how reportportal --project=baseosqe --launch=test_plan tmt run --all report --how reportportal --project=baseosqe --url=... --token=... + tmt run --all report --how reportportal --project=baseosqe --exclude-variables="^(TMT|PACKIT|TESTING_FARM).*" + tmt run -a report -h reportportal --merge --launch "Errata" --attributes "errata:123456" - | # Use ReportPortal as the default report for given plan report: diff --git a/tmt/base.py b/tmt/base.py index 12b2da765f..5b06f5f023 100644 --- a/tmt/base.py +++ b/tmt/base.py @@ -3194,6 +3194,7 @@ class RunData(SerializableContainer): steps: list[str] environment: EnvironmentType remove: bool + rp_uuid: Optional[str] class Run(tmt.utils.Common): @@ -3231,6 +3232,7 @@ def __init__(self, self._environment_from_workdir: EnvironmentType = {} self._environment_from_options: Optional[EnvironmentType] = None self.remove = self.opt('remove') + self.rp_uuid = self.opt('rp-uuid') @tmt.utils.cached_property def runner(self) -> 'tmt.steps.provision.local.GuestLocal': @@ -3307,7 +3309,8 @@ def save(self) -> None: plans=[plan.name for plan in self._plans] if self._plans is not None else None, steps=list(self._cli_context_object.steps), environment=self.environment, - remove=self.remove + remove=self.remove, + rp_uuid=self.rp_uuid ) self.write(Path('run.yaml'), tmt.utils.dict_to_yaml(data.to_serialized())) diff --git a/tmt/schemas/report/reportportal.yaml b/tmt/schemas/report/reportportal.yaml index 184433ba13..ac3517e1db 100644 --- a/tmt/schemas/report/reportportal.yaml +++ b/tmt/schemas/report/reportportal.yaml @@ -34,6 +34,11 @@ properties: launch: type: string + attributes: + type: array + items: + type: string + exclude-variables: type: string diff --git a/tmt/steps/report/reportportal.py b/tmt/steps/report/reportportal.py index c1d7570654..f6cbdd5e1b 100644 --- a/tmt/steps/report/reportportal.py +++ b/tmt/steps/report/reportportal.py @@ -1,7 +1,7 @@ import dataclasses import os import re -from typing import Optional +from typing import List, Optional import requests @@ -15,12 +15,12 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): url: Optional[str] = field( option="--url", metavar="URL", - default=os.environ.get("TMT_REPORT_REPORTPORTAL_URL"), + default=None, help="The URL of the ReportPortal instance where the data should be sent to.") token: Optional[str] = field( option="--token", metavar="TOKEN", - default=os.environ.get("TMT_REPORT_REPORTPORTAL_TOKEN"), + default=None, help="The token to use for upload to the ReportPortal instance (from the user profile).") project: Optional[str] = field( option="--project", @@ -36,27 +36,46 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): option="--exclude-variables", metavar="PATTERN", default="^TMT_.*", - help=""" - Regular expression for excluding environment variables from reporting to ReportPortal - ('^TMT_.*' used by default). - """) + help="Regular expression for excluding environment variables " + "from reporting to ReportPortal ('^TMT_.*' used by default).") + attributes: List[str] = field( + default_factory=list, + multiple=True, + normalize=tmt.utils.normalize_string_list, + option="--attributes", + metavar="KEY:VALUE", + help="Additional attributes to be reported to ReportPortal," + "especially launch attributes for merge option.") + merge: bool = field( + option="--merge", + default=False, + is_flag=True, + help="Report suite per plan and merge them into one launch.") + uuid: Optional[str] = field( + option="--uuid", + metavar="LAUNCH_UUID", + default=None, + help="The launch uuid for additional merging to an existing launch.") + launch_url: str = "" + launch_uuid: str = "" @tmt.steps.provides_method("reportportal") -class ReportReportPortal(tmt.steps.report.ReportPlugin[ReportReportPortalData]): +class ReportReportPortal(tmt.steps.report.ReportPlugin): """ Report test results to a ReportPortal instance via API. - Requires a ``token`` for authentication, a URL of the ReportPortal - instance and the ``project`` name. In addition to command line options - it's possible to use environment variables to set the URL and token: + Requires a token for authentication, a URL of the ReportPortal + instance and the project name. In addition to command line options + it's possible to use environment variables in form of + ``TMT_PLUGIN_REPORT_REPORTPORTAL_${OPTION}``: .. code-block:: bash - export TMT_REPORT_REPORTPORTAL_URL=... - export TMT_REPORT_REPORTPORTAL_TOKEN=... + export TMT_PLUGIN_REPORT_REPORTPORTAL_URL=... + export TMT_PLUGIN_REPORT_REPORTPORTAL_TOKEN=... - The optional ``launch`` name doesn't have to be provided if it is the + The optional launch name doesn't have to be provided if it is the same as the plan name (by default). Assuming the URL and token are provided by the environment variables, the plan config can look like this: @@ -95,19 +114,19 @@ class ReportReportPortal(tmt.steps.report.ReportPlugin[ReportReportPortalData]): def handle_response(self, response: requests.Response) -> None: """ - Check the server response and raise an exception if needed. + Check the endpoint response and raise an exception if needed. """ if not response.ok: raise tmt.utils.ReportError( f"Received non-ok status code from ReportPortal: {response.text}") - self.debug("Response code from the server", response.status_code) - self.debug("Message from the server", response.text) + self.debug("Response code from the endpoint", str(response.status_code)) + self.debug("Message from the endpoint", str(response.text)) def go(self) -> None: """ - Report test results to the server + Report test results to the endpoint Create a ReportPortal launch and its test items, fill it with all parts needed and report the logs. @@ -115,56 +134,110 @@ def go(self) -> None: super().go() - server = self.data.url - if not server: - raise tmt.utils.ReportError("No ReportPortal server url provided.") - server = server.rstrip("/") + endpoint = self.get("url", os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_URL')) + if not endpoint: + raise tmt.utils.ReportError("No ReportPortal endpoint url provided.") + endpoint = endpoint.rstrip("/") - project = self.data.project + project = self.get("project", os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_PROJECT')) if not project: raise tmt.utils.ReportError("No ReportPortal project provided.") - token = self.data.token + token = self.get("token", os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_TOKEN')) if not token: raise tmt.utils.ReportError("No ReportPortal token provided.") - assert self.step.plan.name is not None - launch_name = self.data.launch or self.step.plan.name - - url = f"{server}/api/{self.DEFAULT_API_VERSION}/{project}" + url = f"{endpoint}/api/{self.DEFAULT_API_VERSION}/{project}" headers = { "Authorization": "bearer " + token, "accept": "*/*", "Content-Type": "application/json"} - envar_pattern = self.data.exclude_variables + launch_time = self.step.plan.execute.results()[0].starttime + merge_bool = self.get("merge") + rerun_bool = False + create_launch = True + launch_uuid = "" + suite_uuid = "" + launch_url = "" + launch_name = "" + + envar_pattern = self.get("exclude-variables") or "$^" + extra_attributes = self.get("attributes") + launch_attributes = [ + {'key': attribute.split(':', 2)[0], 'value': attribute.split(':', 2)[1]} + for attribute in extra_attributes] or [] + attributes = [ {'key': key, 'value': value[0]} for key, value in self.step.plan._fmf_context.items()] - launch_time = self.step.plan.execute.results()[0].start_time + for attr in launch_attributes: + if attr not in attributes: + attributes.append(attr) # Communication with RP instance with tmt.utils.retry_session() as session: - # Create a launch - self.info("launch", launch_name, color="cyan") - response = session.post( - url=f"{url}/launch", - headers=headers, - json={ - "name": launch_name, - "description": self.step.plan.summary, - "attributes": attributes, - "startTime": launch_time}) - self.handle_response(response) - launch_uuid = yaml_to_dict(response.text).get("id") - assert launch_uuid is not None + stored_launch_uuid = self.get("uuid") or self.step.plan.my_run.rp_uuid + if merge_bool and stored_launch_uuid: + create_launch = False + launch_uuid = stored_launch_uuid + response = session.get( + url=f"{url}/launch/uuid/{launch_uuid}", + headers=headers) + self.handle_response(response) + launch_name = yaml_to_dict(response.text).get("name") + self.verbose("launch", launch_name, color="yellow") + launch_id = yaml_to_dict(response.text).get("id") + launch_url = f"{endpoint}/ui/#{project}/launches/all/{launch_id}" + else: + # create_launch = True + launch_name = self.get("launch", os.getenv( + 'TMT_PLUGIN_REPORT_REPORTPORTAL_LAUNCH')) or self.step.plan.name + + # Create a launch + self.info("launch", launch_name, color="cyan") + response = session.post( + url=f"{url}/launch", + headers=headers, + json={ + "name": launch_name, + "description": "" if merge_bool else self.step.plan.summary, + "attributes": launch_attributes if merge_bool else attributes, + "startTime": launch_time, + "rerun": rerun_bool}) + self.handle_response(response) + launch_uuid = yaml_to_dict(response.text).get("id") + assert launch_uuid is not None + if merge_bool: + self.step.plan.my_run.rp_uuid = launch_uuid + self.verbose("uuid", launch_uuid, "yellow", shift=1) + self.data.launch_uuid = launch_uuid + + if merge_bool: + # Create a suite + suite_name = self.step.plan.name + self.info("suite", suite_name, color="cyan") + response = session.post( + url=f"{url}/item", + headers=headers, + json={ + "name": suite_name, + "description": self.step.plan.summary, + "attributes": attributes, + "startTime": launch_time, + "launchUuid": launch_uuid, + "type": "suite"}) + self.handle_response(response) + suite_uuid = yaml_to_dict(response.text).get("id") + assert suite_uuid is not None + self.verbose("uuid", suite_uuid, "yellow", shift=1) # For each test for result in self.step.plan.execute.results(): - test = next(test for test in self.step.plan.discover.tests() - if test.serial_number == result.serial_number) + test = [test for test in self.step.plan.discover.tests() + if test.serialnumber == result.serialnumber][0] # TODO: for happz, connect Test to Result if possible item_attributes = attributes.copy() @@ -178,7 +251,7 @@ def go(self) -> None: # Create a test item self.info("test", result.name, color="cyan") response = session.post( - url=f"{url}/item", + url=f"{url}/item{f'/{suite_uuid}' if merge_bool else ''}", headers=headers, json={ "name": result.name, @@ -189,7 +262,7 @@ def go(self) -> None: "launchUuid": launch_uuid, "type": "step", "testCaseId": test.id or None, - "startTime": result.start_time}) + "startTime": result.starttime}) self.handle_response(response) item_uuid = yaml_to_dict(response.text).get("id") assert item_uuid is not None @@ -228,7 +301,7 @@ def go(self) -> None: "itemUuid": item_uuid, "launchUuid": launch_uuid, "level": "ERROR", - "time": result.end_time}) + "time": result.endtime}) self.handle_response(response) # Finish the test item @@ -237,16 +310,29 @@ def go(self) -> None: headers=headers, json={ "launchUuid": launch_uuid, - "endTime": result.end_time, + "endTime": result.endtime, "status": status}) self.handle_response(response) - launch_time = result.end_time - - # Finish the launch - response = session.put( - url=f"{url}/launch/{launch_uuid}/finish", - headers=headers, - json={"endTime": launch_time}) - self.handle_response(response) - link = yaml_to_dict(response.text).get("link") - self.info("url", link, "magenta") + launch_time = result.endtime + + if merge_bool: + # Finish the test suite + response = session.put( + url=f"{url}/item/{suite_uuid}", + headers=headers, + json={ + "launchUuid": launch_uuid, + "endTime": launch_time}) + self.handle_response(response) + + if create_launch: + # Finish the launch + response = session.put( + url=f"{url}/launch/{launch_uuid}/finish", + headers=headers, + json={"endTime": launch_time}) + self.handle_response(response) + launch_url = yaml_to_dict(response.text).get("link") + + self.info("url", launch_url, "magenta") + self.data.launch_url = launch_url From e3b578bc154dbb0261e0125da5404ecaa2b66511 Mon Sep 17 00:00:00 2001 From: Natalia Bubakova Date: Wed, 6 Sep 2023 13:08:14 +0200 Subject: [PATCH 2/7] Small updates of ReportPortal plugin extension * Fixed the environment variables * Prepared defect type locator for implementation of continuous update (idle) * Prepared rerun for implementation of launch update (retry) --- tmt/steps/report/reportportal.py | 37 +++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/tmt/steps/report/reportportal.py b/tmt/steps/report/reportportal.py index f6cbdd5e1b..72b519df12 100644 --- a/tmt/steps/report/reportportal.py +++ b/tmt/steps/report/reportportal.py @@ -15,22 +15,22 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): url: Optional[str] = field( option="--url", metavar="URL", - default=None, + default=os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_URL'), help="The URL of the ReportPortal instance where the data should be sent to.") token: Optional[str] = field( option="--token", metavar="TOKEN", - default=None, + default=os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_TOKEN'), help="The token to use for upload to the ReportPortal instance (from the user profile).") project: Optional[str] = field( option="--project", metavar="PROJECT_NAME", - default=None, + default=os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_PROJECT'), help="Name of the project into which the results should be uploaded.") launch: Optional[str] = field( option="--launch", metavar="LAUNCH_NAME", - default=None, + default=os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_LAUNCH'), help="The launch name (name of plan per launch is used by default).") exclude_variables: str = field( option="--exclude-variables", @@ -134,16 +134,16 @@ def go(self) -> None: super().go() - endpoint = self.get("url", os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_URL')) + endpoint = self.get("url") if not endpoint: raise tmt.utils.ReportError("No ReportPortal endpoint url provided.") endpoint = endpoint.rstrip("/") - project = self.get("project", os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_PROJECT')) + project = self.get("project") if not project: raise tmt.utils.ReportError("No ReportPortal project provided.") - token = self.get("token", os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_TOKEN')) + token = self.get("token") if not token: raise tmt.utils.ReportError("No ReportPortal token provided.") @@ -155,7 +155,10 @@ def go(self) -> None: launch_time = self.step.plan.execute.results()[0].starttime merge_bool = self.get("merge") + rerun_bool = False + # TODO: implement rerun/retry + create_launch = True launch_uuid = "" suite_uuid = "" @@ -178,6 +181,20 @@ def go(self) -> None: # Communication with RP instance with tmt.utils.retry_session() as session: + # get defect type locator + response = session.get( + url=f"{url}/settings", + headers=headers) + self.handle_response(response) + defect_types = yaml_to_dict(response.text).get("subTypes") + dt_tmp = [dt['locator'] + for dt in defect_types['TO_INVESTIGATE'] if dt['longName'] == 'Idle'] + dt_locator = dt_tmp[0] if dt_tmp else None + dt_locator = None + # TODO: + # implement 'idle - update' + # cover cases when there is no Idle defect type defined + stored_launch_uuid = self.get("uuid") or self.step.plan.my_run.rp_uuid if merge_bool and stored_launch_uuid: create_launch = False @@ -192,8 +209,7 @@ def go(self) -> None: launch_url = f"{endpoint}/ui/#{project}/launches/all/{launch_id}" else: # create_launch = True - launch_name = self.get("launch", os.getenv( - 'TMT_PLUGIN_REPORT_REPORTPORTAL_LAUNCH')) or self.step.plan.name + launch_name = self.get("launch") or self.step.plan.name # Create a launch self.info("launch", launch_name, color="cyan") @@ -311,7 +327,8 @@ def go(self) -> None: json={ "launchUuid": launch_uuid, "endTime": result.endtime, - "status": status}) + "status": status, + "issue": {"issueType": dt_locator or "ti001"}}) self.handle_response(response) launch_time = result.endtime From 431d5b77443b0be1cf83cd4041dac66affde2255 Mon Sep 17 00:00:00 2001 From: Natalia Bubakova Date: Wed, 20 Sep 2023 20:51:58 +0200 Subject: [PATCH 3/7] Rewriting ReportPortal plugin extension [WIP]: * mapping according to options --launch-per-plan and --suite-per-plan * uploading to existing launch/suite with options --upload-to-launch LAUNCH_ID, --upload-to-suite SUITE_ID * option --launch-description and preparation for --launch-attributes (for suite-per-plan mapping) * trial option --launch-rerun * trial option --defect-type --- tmt/steps/report/reportportal.py | 205 ++++++++++++++++++++----------- 1 file changed, 134 insertions(+), 71 deletions(-) diff --git a/tmt/steps/report/reportportal.py b/tmt/steps/report/reportportal.py index 72b519df12..c17010dcb0 100644 --- a/tmt/steps/report/reportportal.py +++ b/tmt/steps/report/reportportal.py @@ -32,32 +32,65 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): metavar="LAUNCH_NAME", default=os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_LAUNCH'), help="The launch name (name of plan per launch is used by default).") + launch_description: Optional[str] = field( + option="--launch-description", + metavar="LAUNCH_DESCRIPTION", + default=os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_LAUNCH_DESCRIPTION'), + help="Pass the description for ReportPortal launch, especially with '--suite-per-plan' " + "option (Otherwise Summary from plan fmf data per each launch is used by default).") + ## launch_attributes: List[str] = field( + ## default_factory=list, + ## multiple=True, + ## normalize=tmt.utils.normalize_string_list, + ## option="--launch-attributes", + ## metavar="KEY:VALUE", + ## help="Additional attributes to be reported to ReportPortal," + ## "especially launch attributes for merge option.") + launch_per_plan: bool = field( + option="--launch-per-plan", + default=False, + is_flag=True, + help="Mapping launch per plan, creating one or more launches with no suite structure.") + suite_per_plan: bool = field( + option="--suite-per-plan", + default=False, + is_flag=True, + help="Mapping suite per plan, creating one launch and continous uploading suites into it. " + "Can be used with '--upload-to-launch' option to avoid creating a new launch.") + upload_to_launch: Optional[str] = field( + option="--upload-to-launch", + metavar="LAUNCH_ID", + default=None, + help="Pass the launch ID for an additional test/suite upload to an existing launch. " + "ID can be found in the launch URL.") + upload_to_suite: Optional[str] = field( + option="--upload-to-suite", + metavar="LAUNCH_SUITE", + default=None, + help="Pass the suite ID for an additional test upload to an existing launch. " + "ID can be found in the suite URL.") + launch_rerun: bool = field( + option="--launch-rerun", + default=False, + is_flag=True, + help="Rerun the launch and create Retry version per each test. Note that mapping is " + "based on unique suite/test names and works with '--suite-per-plan' option only.") + defect_type: Optional[str] = field( + option="--defect-type", + metavar="DEFECT_NAME", + default=None, + help="Pass the defect type to be used for failed test " + "('To Investigate' is used by default).") exclude_variables: str = field( option="--exclude-variables", metavar="PATTERN", default="^TMT_.*", help="Regular expression for excluding environment variables " "from reporting to ReportPortal ('^TMT_.*' used by default).") - attributes: List[str] = field( - default_factory=list, - multiple=True, - normalize=tmt.utils.normalize_string_list, - option="--attributes", - metavar="KEY:VALUE", - help="Additional attributes to be reported to ReportPortal," - "especially launch attributes for merge option.") - merge: bool = field( - option="--merge", - default=False, - is_flag=True, - help="Report suite per plan and merge them into one launch.") - uuid: Optional[str] = field( - option="--uuid", - metavar="LAUNCH_UUID", - default=None, - help="The launch uuid for additional merging to an existing launch.") launch_url: str = "" launch_uuid: str = "" + suite_uuid: str = "" + test_uuids: List[str] = [] @tmt.steps.provides_method("reportportal") @@ -99,6 +132,7 @@ class ReportReportPortal(tmt.steps.report.ReportPlugin): aggregation for ReportPortal item if tmt id is not provided. Other reported fmf data are summary, id, web link and contact per test. """ + # TODO: Finish the description ^ with the new options _data_class = ReportReportPortalData @@ -154,84 +188,106 @@ def go(self) -> None: "Content-Type": "application/json"} launch_time = self.step.plan.execute.results()[0].starttime - merge_bool = self.get("merge") - rerun_bool = False - # TODO: implement rerun/retry - create_launch = True - launch_uuid = "" - suite_uuid = "" + + # Create launch, suites (if "--suite_per_plan") and tests; + # or report to existing launch/suite if its id is given + + # TODO: + # * upload_to_launch: change id to uuid + # * launch_uuid: add the param per plan, and do the matching when uploading + # * launch_uuid: add the param per plan, and do the matching when uploading + + launch_uuid = self.get("launch_uuid") or self.step.plan.my_run.rp_uuid + suite_uuid = self.get("suite_uuid") + + launch_id = self.get("upload_to_launch") + suite_id = self.get("upload_to_suite") + + suite_per_plan = self.get("suite_per_plan") + launch_per_plan = self.get("launch_per_plan") + if not launch_per_plan and not suite_per_plan: + launch_per_plan = True # default + elif launch_per_plan and suite_per_plan: + raise tmt.utils.ReportError("The options '--launch-per-plan' and " + "'--suite-per-plan' are mutually exclusive. Choose one of them only.") + + create_launch = not (launch_uuid or launch_id) and not suite_uuid + create_suite = suite_per_plan and not (suite_uuid or suite_id) + launch_url = "" - launch_name = "" + launch_name = self.get("launch") or self.step.plan.name + launch_rerun = self.get("launch_rerun") envar_pattern = self.get("exclude-variables") or "$^" - extra_attributes = self.get("attributes") - launch_attributes = [ - {'key': attribute.split(':', 2)[0], 'value': attribute.split(':', 2)[1]} - for attribute in extra_attributes] or [] + defect_type = self.get("defect_type") attributes = [ {'key': key, 'value': value[0]} for key, value in self.step.plan._fmf_context.items()] - for attr in launch_attributes: - if attr not in attributes: - attributes.append(attr) + + if suite_per_plan: + launch_attributes = "" + # TODO: get common attributes from all plans + else: + launch_attributes = attributes.copy() + + launch_description = self.get("launch_description") or self.step.plan.summary # Communication with RP instance with tmt.utils.retry_session() as session: # get defect type locator - response = session.get( - url=f"{url}/settings", - headers=headers) - self.handle_response(response) - defect_types = yaml_to_dict(response.text).get("subTypes") - dt_tmp = [dt['locator'] - for dt in defect_types['TO_INVESTIGATE'] if dt['longName'] == 'Idle'] - dt_locator = dt_tmp[0] if dt_tmp else None dt_locator = None - # TODO: - # implement 'idle - update' - # cover cases when there is no Idle defect type defined - - stored_launch_uuid = self.get("uuid") or self.step.plan.my_run.rp_uuid - if merge_bool and stored_launch_uuid: - create_launch = False - launch_uuid = stored_launch_uuid - response = session.get( - url=f"{url}/launch/uuid/{launch_uuid}", - headers=headers) + if defect_type: + response = session.get(url=f"{url}/settings", headers=headers) self.handle_response(response) - launch_name = yaml_to_dict(response.text).get("name") - self.verbose("launch", launch_name, color="yellow") - launch_id = yaml_to_dict(response.text).get("id") - launch_url = f"{endpoint}/ui/#{project}/launches/all/{launch_id}" - else: - # create_launch = True - launch_name = self.get("launch") or self.step.plan.name + defect_types = yaml_to_dict(response.text).get("subTypes") + dt_tmp = [dt['locator'] + for dt in defect_types['TO_INVESTIGATE'] if dt['longName'] == defect_type] + dt_locator = dt_tmp[0] if dt_tmp else None + # TODO: check the case when the given defect type is not defined + + if create_launch: # Create a launch self.info("launch", launch_name, color="cyan") response = session.post( - url=f"{url}/launch", - headers=headers, - json={ - "name": launch_name, - "description": "" if merge_bool else self.step.plan.summary, - "attributes": launch_attributes if merge_bool else attributes, - "startTime": launch_time, - "rerun": rerun_bool}) + url=f"{url}/launch", + headers=headers, + json={ "name": launch_name, + "description": launch_description, + "attributes": launch_attributes, + "startTime": launch_time, + "rerun": launch_rerun}) self.handle_response(response) launch_uuid = yaml_to_dict(response.text).get("id") - assert launch_uuid is not None - if merge_bool: + if suite_per_plan: self.step.plan.my_run.rp_uuid = launch_uuid + else: + # Get the launch_uuid or info to log + + # TODO: get launch_uuid from launch_id/suite_id + # if launch_id: + # response = ... + # elif suite_ide: + # response = ... + response = session.get( + url=f"{url}/launch/uuid/{launch_uuid}", + headers=headers) + self.handle_response(response) + launch_name = yaml_to_dict(response.text).get("name") + self.verbose("launch", launch_name, color="yellow") + launch_id = yaml_to_dict(response.text).get("id") + launch_url = f"{endpoint}/ui/#{project}/launches/all/{launch_id}" + + assert launch_uuid is not None self.verbose("uuid", launch_uuid, "yellow", shift=1) self.data.launch_uuid = launch_uuid - if merge_bool: + if create_suite: # Create a suite suite_name = self.step.plan.name self.info("suite", suite_name, color="cyan") @@ -267,7 +323,7 @@ def go(self) -> None: # Create a test item self.info("test", result.name, color="cyan") response = session.post( - url=f"{url}/item{f'/{suite_uuid}' if merge_bool else ''}", + url=f"{url}/item{f'/{suite_uuid}' if suite_uuid else ''}", headers=headers, json={ "name": result.name, @@ -320,6 +376,8 @@ def go(self) -> None: "time": result.endtime}) self.handle_response(response) + # TODO: Add tmt files as attachments + # Finish the test item response = session.put( url=f"{url}/item/{item_uuid}", @@ -332,7 +390,7 @@ def go(self) -> None: self.handle_response(response) launch_time = result.endtime - if merge_bool: + if create_suite: # Finish the test suite response = session.put( url=f"{url}/item/{suite_uuid}", @@ -342,6 +400,11 @@ def go(self) -> None: "endTime": launch_time}) self.handle_response(response) + # TODO: Get if it is the last plan + # + # if create_launch and (not suite_per_plan or + # (suite_per_plan and this-is-the-last-plan)): + if create_launch: # Finish the launch response = session.put( From 7746f71ca74d52e2aa7ee169b27e222899853733 Mon Sep 17 00:00:00 2001 From: nbubakov Date: Sat, 18 Nov 2023 01:35:49 +0100 Subject: [PATCH 4/7] Rewriting ReportPortal plugin extension [WIP in its final stages] * functional --suite-per-plan option that uploads all plans into a launch; + reporting common atributes from all plans, closing the launch after last plan * additional upload of tests/suites into a existing launch * minor fixing of previously implemented (--defect-type, etc.) * listed all the points to be done yet so it is ready for review --- tmt/steps/report/reportportal.py | 140 ++++++++++++++++++------------- 1 file changed, 81 insertions(+), 59 deletions(-) diff --git a/tmt/steps/report/reportportal.py b/tmt/steps/report/reportportal.py index c17010dcb0..50eaac21e6 100644 --- a/tmt/steps/report/reportportal.py +++ b/tmt/steps/report/reportportal.py @@ -38,14 +38,6 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): default=os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_LAUNCH_DESCRIPTION'), help="Pass the description for ReportPortal launch, especially with '--suite-per-plan' " "option (Otherwise Summary from plan fmf data per each launch is used by default).") - ## launch_attributes: List[str] = field( - ## default_factory=list, - ## multiple=True, - ## normalize=tmt.utils.normalize_string_list, - ## option="--launch-attributes", - ## metavar="KEY:VALUE", - ## help="Additional attributes to be reported to ReportPortal," - ## "especially launch attributes for merge option.") launch_per_plan: bool = field( option="--launch-per-plan", default=False, @@ -56,6 +48,7 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): default=False, is_flag=True, help="Mapping suite per plan, creating one launch and continous uploading suites into it. " + "Recommended to use with '--launch' and '--launch-description' options." "Can be used with '--upload-to-launch' option to avoid creating a new launch.") upload_to_launch: Optional[str] = field( option="--upload-to-launch", @@ -79,8 +72,10 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): option="--defect-type", metavar="DEFECT_NAME", default=None, - help="Pass the defect type to be used for failed test " - "('To Investigate' is used by default).") + help="Pass the defect type to be used for failed test, which is defined in the project" + " (e.g. 'Idle'). 'To Investigate' is used by default.") + # TODO: test how to create empty test skeleton, all with Idle defect_type + # (as it reports defect_type only when it fails) exclude_variables: str = field( option="--exclude-variables", metavar="PATTERN", @@ -90,7 +85,7 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): launch_url: str = "" launch_uuid: str = "" suite_uuid: str = "" - test_uuids: List[str] = [] + test_uuids: List[str] = "" @tmt.steps.provides_method("reportportal") @@ -166,6 +161,15 @@ def go(self) -> None: fill it with all parts needed and report the logs. """ + # TODO: + # * resolve the problem with mypy + # * replace rp_uuid with an universal structure for tmt plugin data + # * check the problem with --upload-to-suite functonality + # * check the param per plan, and do the matching when uploading (launch_uuid) + # * check the defect_type, idle functionality + # * upload documentation (help and spec) + # * add the tests for new features + super().go() endpoint = self.get("url") @@ -187,20 +191,12 @@ def go(self) -> None: "accept": "*/*", "Content-Type": "application/json"} - launch_time = self.step.plan.execute.results()[0].starttime - - + launch_time = self.step.plan.execute.results()[0].start_time # Create launch, suites (if "--suite_per_plan") and tests; # or report to existing launch/suite if its id is given - - # TODO: - # * upload_to_launch: change id to uuid - # * launch_uuid: add the param per plan, and do the matching when uploading - # * launch_uuid: add the param per plan, and do the matching when uploading - launch_uuid = self.get("launch_uuid") or self.step.plan.my_run.rp_uuid - suite_uuid = self.get("suite_uuid") + suite_uuid = self.get("suite_uuid") launch_id = self.get("upload_to_launch") suite_id = self.get("upload_to_suite") @@ -208,11 +204,12 @@ def go(self) -> None: suite_per_plan = self.get("suite_per_plan") launch_per_plan = self.get("launch_per_plan") if not launch_per_plan and not suite_per_plan: - launch_per_plan = True # default + launch_per_plan = True # by default elif launch_per_plan and suite_per_plan: - raise tmt.utils.ReportError("The options '--launch-per-plan' and " - "'--suite-per-plan' are mutually exclusive. Choose one of them only.") - + raise tmt.utils.ReportError( + "The options '--launch-per-plan' and " + "'--suite-per-plan' are mutually exclusive. Choose one of them only.") + create_launch = not (launch_uuid or launch_id) and not suite_uuid create_suite = suite_per_plan and not (suite_uuid or suite_id) @@ -228,8 +225,19 @@ def go(self) -> None: for key, value in self.step.plan._fmf_context.items()] if suite_per_plan: - launch_attributes = "" - # TODO: get common attributes from all plans + # Get common attributes from all plans + merged_plans = [{key: value[0] for key, value in plan._fmf_context.items()} + for plan in self.step.plan.my_run.plans] + result_dict = merged_plans[0] + for current_plan in merged_plans[1:]: + tmp_dict = {} + for key, value in current_plan.items(): + if key in result_dict and result_dict[key] == value: + tmp_dict[key] = value + result_dict = tmp_dict + launch_attributes = [ + {'key': key, 'value': value} + for key, value in result_dict.items()] else: launch_attributes = attributes.copy() @@ -238,49 +246,62 @@ def go(self) -> None: # Communication with RP instance with tmt.utils.retry_session() as session: - # get defect type locator + # Get defect type locator dt_locator = None if defect_type: response = session.get(url=f"{url}/settings", headers=headers) self.handle_response(response) defect_types = yaml_to_dict(response.text).get("subTypes") - dt_tmp = [dt['locator'] - for dt in defect_types['TO_INVESTIGATE'] if dt['longName'] == defect_type] + dt_tmp = [dt['locator'] for dt in defect_types['TO_INVESTIGATE'] + if dt['longName'].lower() == defect_type.lower()] dt_locator = dt_tmp[0] if dt_tmp else None - # TODO: check the case when the given defect type is not defined + if not dt_locator: + raise tmt.utils.ReportError( + f"Defect type {defect_type} may not be defined in the project {project}") + self.verbose(f"defect_type{defect_type}", dt_locator) if create_launch: # Create a launch self.info("launch", launch_name, color="cyan") response = session.post( - url=f"{url}/launch", - headers=headers, - json={ "name": launch_name, - "description": launch_description, - "attributes": launch_attributes, - "startTime": launch_time, - "rerun": launch_rerun}) + url=f"{url}/launch", + headers=headers, + json={"name": launch_name, + "description": launch_description, + "attributes": launch_attributes, + "startTime": launch_time, + "rerun": launch_rerun}) self.handle_response(response) launch_uuid = yaml_to_dict(response.text).get("id") if suite_per_plan: self.step.plan.my_run.rp_uuid = launch_uuid + else: # Get the launch_uuid or info to log + if launch_id: + response = session.get( + url=f"{url}/launch/{launch_id}", + headers=headers) + self.handle_response(response) + launch_uuid = yaml_to_dict(response.text).get("uuid") - # TODO: get launch_uuid from launch_id/suite_id - # if launch_id: - # response = ... - # elif suite_ide: - # response = ... + elif suite_id: + response = session.get( + url=f"{url}/item/{suite_id}", + headers=headers) + self.handle_response(response) + suite_uuid = yaml_to_dict(response.text).get("uuid") + + elif launch_uuid: + response = session.get( + url=f"{url}/launch/uuid/{launch_uuid}", + headers=headers) + self.handle_response(response) + launch_id = yaml_to_dict(response.text).get("id") - response = session.get( - url=f"{url}/launch/uuid/{launch_uuid}", - headers=headers) - self.handle_response(response) launch_name = yaml_to_dict(response.text).get("name") self.verbose("launch", launch_name, color="yellow") - launch_id = yaml_to_dict(response.text).get("id") launch_url = f"{endpoint}/ui/#{project}/launches/all/{launch_id}" assert launch_uuid is not None @@ -308,8 +329,8 @@ def go(self) -> None: # For each test for result in self.step.plan.execute.results(): - test = [test for test in self.step.plan.discover.tests() - if test.serialnumber == result.serialnumber][0] + test = next((test for test in self.step.plan.discover.tests() + if test.serial_number == result.serial_number), None) # TODO: for happz, connect Test to Result if possible item_attributes = attributes.copy() @@ -334,7 +355,7 @@ def go(self) -> None: "launchUuid": launch_uuid, "type": "step", "testCaseId": test.id or None, - "startTime": result.starttime}) + "startTime": result.start_time}) self.handle_response(response) item_uuid = yaml_to_dict(response.text).get("id") assert item_uuid is not None @@ -373,7 +394,7 @@ def go(self) -> None: "itemUuid": item_uuid, "launchUuid": launch_uuid, "level": "ERROR", - "time": result.endtime}) + "time": result.end_time}) self.handle_response(response) # TODO: Add tmt files as attachments @@ -384,11 +405,11 @@ def go(self) -> None: headers=headers, json={ "launchUuid": launch_uuid, - "endTime": result.endtime, + "endTime": result.end_time, "status": status, "issue": {"issueType": dt_locator or "ti001"}}) self.handle_response(response) - launch_time = result.endtime + launch_time = result.end_time if create_suite: # Finish the test suite @@ -400,12 +421,12 @@ def go(self) -> None: "endTime": launch_time}) self.handle_response(response) - # TODO: Get if it is the last plan - # - # if create_launch and (not suite_per_plan or - # (suite_per_plan and this-is-the-last-plan)): + is_the_last_plan = self.step.plan == self.step.plan.my_run.plans[-1] + is_additional_upload = self.get("upload_to_launch") is None \ + or self.get("upload_to_suite") is None - if create_launch: + if (create_launch and not suite_per_plan) or \ + (suite_per_plan and is_the_last_plan and not is_additional_upload): # Finish the launch response = session.put( url=f"{url}/launch/{launch_uuid}/finish", @@ -414,5 +435,6 @@ def go(self) -> None: self.handle_response(response) launch_url = yaml_to_dict(response.text).get("link") + assert launch_url is not None self.info("url", launch_url, "magenta") self.data.launch_url = launch_url From 2e99f3b1bd23361de6c23cd1ced32c7612aecb05 Mon Sep 17 00:00:00 2001 From: nbubakov Date: Mon, 27 Nov 2023 12:45:40 +0100 Subject: [PATCH 5/7] Rewriting ReportPortal plugin extension [WIP in its final stages] * functional idle report and additional report within run id * functional upload to launch * debug based on option combinations --- tmt/steps/report/reportportal.py | 145 +++++++++++++++++++------------ 1 file changed, 89 insertions(+), 56 deletions(-) diff --git a/tmt/steps/report/reportportal.py b/tmt/steps/report/reportportal.py index 50eaac21e6..8dddddc0ed 100644 --- a/tmt/steps/report/reportportal.py +++ b/tmt/steps/report/reportportal.py @@ -1,7 +1,8 @@ import dataclasses import os import re -from typing import List, Optional +from time import time +from typing import Optional import requests @@ -32,7 +33,7 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): metavar="LAUNCH_NAME", default=os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_LAUNCH'), help="The launch name (name of plan per launch is used by default).") - launch_description: Optional[str] = field( + launch_description: str = field( option="--launch-description", metavar="LAUNCH_DESCRIPTION", default=os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_LAUNCH_DESCRIPTION'), @@ -50,13 +51,13 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): help="Mapping suite per plan, creating one launch and continous uploading suites into it. " "Recommended to use with '--launch' and '--launch-description' options." "Can be used with '--upload-to-launch' option to avoid creating a new launch.") - upload_to_launch: Optional[str] = field( + upload_to_launch: str = field( option="--upload-to-launch", metavar="LAUNCH_ID", default=None, help="Pass the launch ID for an additional test/suite upload to an existing launch. " "ID can be found in the launch URL.") - upload_to_suite: Optional[str] = field( + upload_to_suite: str = field( option="--upload-to-suite", metavar="LAUNCH_SUITE", default=None, @@ -66,8 +67,8 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): option="--launch-rerun", default=False, is_flag=True, - help="Rerun the launch and create Retry version per each test. Note that mapping is " - "based on unique suite/test names and works with '--suite-per-plan' option only.") + help="Rerun the launch based on unique test paths and ids to create Retry item" + "with a new version per each test. Supported in 'suite-per-plan' structure only.") defect_type: Optional[str] = field( option="--defect-type", metavar="DEFECT_NAME", @@ -85,7 +86,7 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): launch_url: str = "" launch_uuid: str = "" suite_uuid: str = "" - test_uuids: List[str] = "" + test_uuids: list[str] = "" @tmt.steps.provides_method("reportportal") @@ -161,7 +162,11 @@ def go(self) -> None: fill it with all parts needed and report the logs. """ - # TODO: + # TODO: ? why it did not closed : + # tmt run discover provision -h connect -g 10.0.185.29 -u root + # -p fo0m4nchU prepare report -h reportportal --suite-per-plan + # --defect-type idle -vvv finish tests -n . + # * test uploading to idle tests # * resolve the problem with mypy # * replace rp_uuid with an universal structure for tmt plugin data # * check the problem with --upload-to-suite functonality @@ -169,6 +174,14 @@ def go(self) -> None: # * check the defect_type, idle functionality # * upload documentation (help and spec) # * add the tests for new features + # * add uploading files + # * edit schemas + + # * check optionality in arguments + # * restrict combinations + set warning + # - forbid rerun && launch-per-plan + # - forbid upload-to-suite && launch-per-plan + # * try read a launch_uuid at first plan (self.step.plan.report.launch_uuid) super().go() @@ -191,7 +204,13 @@ def go(self) -> None: "accept": "*/*", "Content-Type": "application/json"} - launch_time = self.step.plan.execute.results()[0].start_time + launch_time = str(int(time() * 1000)) + + # to support idle tests + executed = False + if len(self.step.plan.execute.results()) > 0: + launch_time = self.step.plan.execute.results()[0].start_time + executed = True # Create launch, suites (if "--suite_per_plan") and tests; # or report to existing launch/suite if its id is given @@ -210,7 +229,7 @@ def go(self) -> None: "The options '--launch-per-plan' and " "'--suite-per-plan' are mutually exclusive. Choose one of them only.") - create_launch = not (launch_uuid or launch_id) and not suite_uuid + create_launch = not (launch_uuid or launch_id or suite_uuid or suite_id) create_suite = suite_per_plan and not (suite_uuid or suite_id) launch_url = "" @@ -257,8 +276,8 @@ def go(self) -> None: dt_locator = dt_tmp[0] if dt_tmp else None if not dt_locator: raise tmt.utils.ReportError( - f"Defect type {defect_type} may not be defined in the project {project}") - self.verbose(f"defect_type{defect_type}", dt_locator) + f"Defect type '{defect_type}' is not be defined in the project {project}") + self.verbose("defect_typ", defect_type, "yellow") if create_launch: @@ -279,19 +298,23 @@ def go(self) -> None: else: # Get the launch_uuid or info to log - if launch_id: + + if suite_id: response = session.get( - url=f"{url}/launch/{launch_id}", + url=f"{url}/item/{suite_id}", headers=headers) self.handle_response(response) - launch_uuid = yaml_to_dict(response.text).get("uuid") + suite_uuid = yaml_to_dict(response.text).get("uuid") + self.info("suite_id", suite_id, color="red") + launch_id = yaml_to_dict(response.text).get("launchId") + self.info("launch_id", launch_id, color="red") - elif suite_id: + if launch_id: response = session.get( - url=f"{url}/item/{suite_id}", + url=f"{url}/launch/{launch_id}", headers=headers) self.handle_response(response) - suite_uuid = yaml_to_dict(response.text).get("uuid") + launch_uuid = yaml_to_dict(response.text).get("uuid") elif launch_uuid: response = session.get( @@ -328,10 +351,15 @@ def go(self) -> None: self.verbose("uuid", suite_uuid, "yellow", shift=1) # For each test - for result in self.step.plan.execute.results(): - test = next((test for test in self.step.plan.discover.tests() - if test.serial_number == result.serial_number), None) + for test in self.step.plan.discover.tests(): + test_time = str(int(time() * 1000)) + if executed: + result = next((result for result in self.step.plan.execute.results() + if test.serial_number == result.serial_number), None) + test_time = result.start_time + # TODO: for happz, connect Test to Result if possible + # (but now it is probably too hackish to be fixed) item_attributes = attributes.copy() if test.contact: @@ -342,12 +370,12 @@ def go(self) -> None: if not re.search(envar_pattern, key)] # Create a test item - self.info("test", result.name, color="cyan") + self.info("test", test.name, color="cyan") response = session.post( url=f"{url}/item{f'/{suite_uuid}' if suite_uuid else ''}", headers=headers, json={ - "name": result.name, + "name": test.name, "description": test.summary, "attributes": item_attributes, "parameters": env_vars, @@ -355,61 +383,66 @@ def go(self) -> None: "launchUuid": launch_uuid, "type": "step", "testCaseId": test.id or None, - "startTime": result.start_time}) + "startTime": test_time}) self.handle_response(response) item_uuid = yaml_to_dict(response.text).get("id") assert item_uuid is not None self.verbose("uuid", item_uuid, "yellow", shift=1) - # For each log - for index, log_path in enumerate(result.log): - try: - log = self.step.plan.execute.read(log_path) - except tmt.utils.FileError: - continue - - level = "INFO" if log_path == result.log[0] else "TRACE" - status = self.TMT_TO_RP_RESULT_STATUS[result.result] - - # Upload log - response = session.post( - url=f"{url}/log/entry", - headers=headers, - json={ - "message": log, - "itemUuid": item_uuid, - "launchUuid": launch_uuid, - "level": level, - "time": result.end_time}) - self.handle_response(response) + # to supoort idle tests + status = "SKIPPED" + if executed: + # For each log + for index, log_path in enumerate(result.log): + try: + log = self.step.plan.execute.read(log_path) + except tmt.utils.FileError: + continue + + level = "INFO" if log_path == result.log[0] else "TRACE" + status = self.TMT_TO_RP_RESULT_STATUS[result.result] - # Write out failures - if index == 0 and status == "FAILED": - message = result.failures(log) + # Upload log response = session.post( url=f"{url}/log/entry", headers=headers, json={ - "message": message, + "message": log, "itemUuid": item_uuid, "launchUuid": launch_uuid, - "level": "ERROR", + "level": level, "time": result.end_time}) self.handle_response(response) + # Write out failures + if index == 0 and status == "FAILED": + message = result.failures(log) + response = session.post( + url=f"{url}/log/entry", + headers=headers, + json={ + "message": message, + "itemUuid": item_uuid, + "launchUuid": launch_uuid, + "level": "ERROR", + "time": result.end_time}) + self.handle_response(response) + # TODO: Add tmt files as attachments + test_time = result.end_time + # Finish the test item response = session.put( url=f"{url}/item/{item_uuid}", headers=headers, json={ "launchUuid": launch_uuid, - "endTime": result.end_time, + "endTime": test_time, "status": status, "issue": {"issueType": dt_locator or "ti001"}}) self.handle_response(response) - launch_time = result.end_time + launch_time = test_time if create_suite: # Finish the test suite @@ -422,11 +455,11 @@ def go(self) -> None: self.handle_response(response) is_the_last_plan = self.step.plan == self.step.plan.my_run.plans[-1] - is_additional_upload = self.get("upload_to_launch") is None \ - or self.get("upload_to_suite") is None + additional_upload = self.get("upload_to_launch") is not None \ + or self.get("upload_to_suite") is not None - if (create_launch and not suite_per_plan) or \ - (suite_per_plan and is_the_last_plan and not is_additional_upload): + if ((launch_per_plan or (suite_per_plan and is_the_last_plan)) + and not additional_upload): # Finish the launch response = session.put( url=f"{url}/launch/{launch_uuid}/finish", From 719f5647d36e419fc7868237288b2bfb131f0920 Mon Sep 17 00:00:00 2001 From: nbubakov Date: Tue, 12 Dec 2023 11:04:12 +0100 Subject: [PATCH 6/7] Rewriting ReportPortal plugin extension * fixed the functonality to report idle tests and additional results * rewritten code into methods * added version * postposing the functionality 'adding files as attanchement' to another PR --- tmt/base.py | 5 +- tmt/steps/report/reportportal.py | 405 +++++++++++++++++-------------- 2 files changed, 226 insertions(+), 184 deletions(-) diff --git a/tmt/base.py b/tmt/base.py index 5b06f5f023..12b2da765f 100644 --- a/tmt/base.py +++ b/tmt/base.py @@ -3194,7 +3194,6 @@ class RunData(SerializableContainer): steps: list[str] environment: EnvironmentType remove: bool - rp_uuid: Optional[str] class Run(tmt.utils.Common): @@ -3232,7 +3231,6 @@ def __init__(self, self._environment_from_workdir: EnvironmentType = {} self._environment_from_options: Optional[EnvironmentType] = None self.remove = self.opt('remove') - self.rp_uuid = self.opt('rp-uuid') @tmt.utils.cached_property def runner(self) -> 'tmt.steps.provision.local.GuestLocal': @@ -3309,8 +3307,7 @@ def save(self) -> None: plans=[plan.name for plan in self._plans] if self._plans is not None else None, steps=list(self._cli_context_object.steps), environment=self.environment, - remove=self.remove, - rp_uuid=self.rp_uuid + remove=self.remove ) self.write(Path('run.yaml'), tmt.utils.dict_to_yaml(data.to_serialized())) diff --git a/tmt/steps/report/reportportal.py b/tmt/steps/report/reportportal.py index 8dddddc0ed..bf228437a7 100644 --- a/tmt/steps/report/reportportal.py +++ b/tmt/steps/report/reportportal.py @@ -6,6 +6,7 @@ import requests +import tmt.log import tmt.steps.report from tmt.result import ResultOutcome from tmt.utils import field, yaml_to_dict @@ -61,8 +62,8 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): option="--upload-to-suite", metavar="LAUNCH_SUITE", default=None, - help="Pass the suite ID for an additional test upload to an existing launch. " - "ID can be found in the suite URL.") + help="Pass the suite ID for an additional test upload to a suite " + "within an existing launch. ID can be found in the suite URL.") launch_rerun: bool = field( option="--launch-rerun", default=False, @@ -82,11 +83,17 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): metavar="PATTERN", default="^TMT_.*", help="Regular expression for excluding environment variables " - "from reporting to ReportPortal ('^TMT_.*' used by default).") + "from reporting to ReportPortal ('^TMT_.*' used by default)." + "Parameters in ReportPortal get filtered out by the pattern" + "to prevent overloading and to preserve the history aggregation" + "for ReportPortal item if tmt id is not provided") + launch_url: str = "" launch_uuid: str = "" suite_uuid: str = "" - test_uuids: list[str] = "" + test_uuids: dict[int, str] = field( + default_factory=dict + ) @tmt.steps.provides_method("reportportal") @@ -94,15 +101,22 @@ class ReportReportPortal(tmt.steps.report.ReportPlugin): """ Report test results to a ReportPortal instance via API. - Requires a token for authentication, a URL of the ReportPortal - instance and the project name. In addition to command line options - it's possible to use environment variables in form of - ``TMT_PLUGIN_REPORT_REPORTPORTAL_${OPTION}``: + For communication with Report Portal API is neccessary to provide + following options: + + * token for authentication + * URL of the ReportPortal instance + * project name + * optional API version to override the default one (v1) + * optional launch name to override the deafult name based on the tmt + plan name + + In addition to command line options it's possible to use environment + variables: .. code-block:: bash - export TMT_PLUGIN_REPORT_REPORTPORTAL_URL=... - export TMT_PLUGIN_REPORT_REPORTPORTAL_TOKEN=... + export TMT_PLUGIN_REPORT_REPORTPORTAL_${MY_OPTION}=${MY_VALUE} The optional launch name doesn't have to be provided if it is the same as the plan name (by default). Assuming the URL and token @@ -121,14 +135,23 @@ class ReportReportPortal(tmt.steps.report.ReportPlugin): environment: ... - The context and environment sections must be filled in order to - report context as attributes and environment variables as parameters - in the Item Details. Environment variables can be filtered out by - pattern to prevent overloading and to preserve the history - aggregation for ReportPortal item if tmt id is not provided. Other - reported fmf data are summary, id, web link and contact per test. + Where the context and environment sections must be filled with + corresponding data in order to report context as attributes + (arch, component, distro, trigger, compose, etc.) and + environment variables as parameters in the Item Details. + + Other reported fmf data are summary, id, web link and contact per + test. + + There are supported two ways of mapping plans into ReportPortal + + * launch-per-plan (default) with reported structure 'launch > test', + resulting in one or more launches. + * suite-per-plan with reported structure 'launch > suite > test' + resulting in one launch only, and one or more suites within. + (Recommended to define launch-name and launch-description in + addition) """ - # TODO: Finish the description ^ with the new options _data_class = ReportReportPortalData @@ -154,6 +177,74 @@ def handle_response(self, response: requests.Response) -> None: self.debug("Response code from the endpoint", str(response.status_code)) self.debug("Message from the endpoint", str(response.text)) + def time(self) -> str: + return str(int(time() * 1000)) + + def get_headers(self) -> dict[str, str]: + return {"Authorization": "bearer " + self.token, + "accept": "*/*", + "Content-Type": "application/json"} + + def get_url(self) -> str: + api_version = os.getenv( + 'TMT_PLUGIN_REPORT_REPORTPORTAL_API_VERSION') or self.DEFAULT_API_VERSION + return f"{self.endpoint}/api/{api_version}/{self.project}" + + def construct_launch_attributes(self, suite_per_plan: bool, + attributes: dict[str, str]) -> dict[str, str]: + if not suite_per_plan: + return attributes.copy() + + # Get common attributes across the plans + merged_plans = [{key: value[0] for key, value in plan._fmf_context.items()} + for plan in self.step.plan.my_run.plans] + result_dict = merged_plans[0] + for current_plan in merged_plans[1:]: + tmp_dict = {} + for key, value in current_plan.items(): + if key in result_dict and result_dict[key] == value: + tmp_dict[key] = value + result_dict = tmp_dict + return [{'key': key, 'value': value} for key, value in result_dict.items()] + + def get_defect_type_locator(self, session: requests.Session, defect_type: str) -> str: + if not defect_type: + return "ti001" + + # Get defect type locator via api + response = self.get_rp_api(session, "settings") + defect_types = yaml_to_dict(response.text).get("subTypes") + dt_tmp = [dt['locator'] for dt in defect_types['TO_INVESTIGATE'] + if dt['longName'].lower() == defect_type.lower()] + dt_locator = dt_tmp[0] if dt_tmp else None + if not dt_locator: + raise tmt.utils.ReportError( + f"Defect type '{defect_type}' is not be defined in the project {self.project}") + self.verbose("defect_typ", defect_type, "yellow") + return dt_locator + + def get_rp_api(self, session: requests.Session, data_path: str) -> str: + response = session.get(url=f"{self.get_url()}/{data_path}", + headers=self.get_headers()) + self.handle_response(response) + return response + + def post_rp_api(self, session: requests.Session, item_path: str, json: dict[str, str]) -> str: + response = session.post( + url=f"{self.get_url()}/{item_path}", + headers=self.get_headers(), + json=json) + self.handle_response(response) + return response + + def put_rp_api(self, session: requests.Session, item_path: str, json: dict[str, str]) -> str: + response = session.put( + url=f"{self.get_url()}/{item_path}", + headers=self.get_headers(), + json=json) + self.handle_response(response) + return response + def go(self) -> None: """ Report test results to the endpoint @@ -162,51 +253,39 @@ def go(self) -> None: fill it with all parts needed and report the logs. """ - # TODO: ? why it did not closed : - # tmt run discover provision -h connect -g 10.0.185.29 -u root - # -p fo0m4nchU prepare report -h reportportal --suite-per-plan - # --defect-type idle -vvv finish tests -n . - # * test uploading to idle tests + # TODO: (to be deleted after review) # * resolve the problem with mypy - # * replace rp_uuid with an universal structure for tmt plugin data # * check the problem with --upload-to-suite functonality # * check the param per plan, and do the matching when uploading (launch_uuid) - # * check the defect_type, idle functionality # * upload documentation (help and spec) - # * add the tests for new features - # * add uploading files + # * add the tests for new features --> another PR + # * add uploading files --> another PR # * edit schemas - # * check optionality in arguments # * restrict combinations + set warning # - forbid rerun && launch-per-plan # - forbid upload-to-suite && launch-per-plan # * try read a launch_uuid at first plan (self.step.plan.report.launch_uuid) + # * rewrite into smarter and neater code (bolean logic for option combinations) super().go() - endpoint = self.get("url") - if not endpoint: + self.endpoint = self.get("url") + if not self.endpoint: raise tmt.utils.ReportError("No ReportPortal endpoint url provided.") - endpoint = endpoint.rstrip("/") + self.endpoint = self.endpoint.rstrip("/") - project = self.get("project") - if not project: + self.project = self.get("project") + if not self.project: raise tmt.utils.ReportError("No ReportPortal project provided.") - token = self.get("token") - if not token: + self.token = self.get("token") + if not self.token: raise tmt.utils.ReportError("No ReportPortal token provided.") - url = f"{endpoint}/api/{self.DEFAULT_API_VERSION}/{project}" - headers = { - "Authorization": "bearer " + token, - "accept": "*/*", - "Content-Type": "application/json"} - - launch_time = str(int(time() * 1000)) + launch_time = self.time() - # to support idle tests + # Supporting idle tests executed = False if len(self.step.plan.execute.results()) > 0: launch_time = self.step.plan.execute.results()[0].start_time @@ -214,12 +293,6 @@ def go(self) -> None: # Create launch, suites (if "--suite_per_plan") and tests; # or report to existing launch/suite if its id is given - launch_uuid = self.get("launch_uuid") or self.step.plan.my_run.rp_uuid - suite_uuid = self.get("suite_uuid") - - launch_id = self.get("upload_to_launch") - suite_id = self.get("upload_to_suite") - suite_per_plan = self.get("suite_per_plan") launch_per_plan = self.get("launch_per_plan") if not launch_per_plan and not suite_per_plan: @@ -229,11 +302,23 @@ def go(self) -> None: "The options '--launch-per-plan' and " "'--suite-per-plan' are mutually exclusive. Choose one of them only.") - create_launch = not (launch_uuid or launch_id or suite_uuid or suite_id) + suite_id = self.get("upload_to_suite") + launch_id = self.get("upload_to_launch") + + suite_uuid = self.get("suite_uuid") + launch_uuid = self.get("launch_uuid") + additional_upload = suite_id or launch_id or launch_uuid + is_the_first_plan = self.step.plan == self.step.plan.my_run.plans[0] + if not launch_uuid and suite_per_plan and not is_the_first_plan: + launch_uuid = self.step.plan.my_run.plans[0].report.data[0].launch_uuid + + create_test = not self.data.test_uuids create_suite = suite_per_plan and not (suite_uuid or suite_id) + create_launch = not (launch_uuid or launch_id or suite_uuid or suite_id) - launch_url = "" launch_name = self.get("launch") or self.step.plan.name + suite_name = "" + launch_url = "" launch_rerun = self.get("launch_rerun") envar_pattern = self.get("exclude-variables") or "$^" @@ -243,121 +328,83 @@ def go(self) -> None: {'key': key, 'value': value[0]} for key, value in self.step.plan._fmf_context.items()] - if suite_per_plan: - # Get common attributes from all plans - merged_plans = [{key: value[0] for key, value in plan._fmf_context.items()} - for plan in self.step.plan.my_run.plans] - result_dict = merged_plans[0] - for current_plan in merged_plans[1:]: - tmp_dict = {} - for key, value in current_plan.items(): - if key in result_dict and result_dict[key] == value: - tmp_dict[key] = value - result_dict = tmp_dict - launch_attributes = [ - {'key': key, 'value': value} - for key, value in result_dict.items()] - else: - launch_attributes = attributes.copy() + launch_attributes = self.construct_launch_attributes(suite_per_plan, attributes) launch_description = self.get("launch_description") or self.step.plan.summary # Communication with RP instance with tmt.utils.retry_session() as session: - # Get defect type locator - dt_locator = None - if defect_type: - response = session.get(url=f"{url}/settings", headers=headers) - self.handle_response(response) - defect_types = yaml_to_dict(response.text).get("subTypes") - dt_tmp = [dt['locator'] for dt in defect_types['TO_INVESTIGATE'] - if dt['longName'].lower() == defect_type.lower()] - dt_locator = dt_tmp[0] if dt_tmp else None - if not dt_locator: - raise tmt.utils.ReportError( - f"Defect type '{defect_type}' is not be defined in the project {project}") - self.verbose("defect_typ", defect_type, "yellow") - if create_launch: # Create a launch self.info("launch", launch_name, color="cyan") - response = session.post( - url=f"{url}/launch", - headers=headers, - json={"name": launch_name, - "description": launch_description, - "attributes": launch_attributes, - "startTime": launch_time, - "rerun": launch_rerun}) - self.handle_response(response) + response = self.post_rp_api(session, "launch", + json={"name": launch_name, + "description": launch_description, + "attributes": launch_attributes, + "startTime": launch_time, + "rerun": launch_rerun}) launch_uuid = yaml_to_dict(response.text).get("id") - if suite_per_plan: - self.step.plan.my_run.rp_uuid = launch_uuid else: # Get the launch_uuid or info to log - if suite_id: - response = session.get( - url=f"{url}/item/{suite_id}", - headers=headers) - self.handle_response(response) + response = self.get_rp_api(session, f"item/{suite_id}") suite_uuid = yaml_to_dict(response.text).get("uuid") - self.info("suite_id", suite_id, color="red") + # self.info("suite_id", suite_id, color="yellow") + suite_name = yaml_to_dict(response.text).get("name") launch_id = yaml_to_dict(response.text).get("launchId") - self.info("launch_id", launch_id, color="red") if launch_id: - response = session.get( - url=f"{url}/launch/{launch_id}", - headers=headers) - self.handle_response(response) + response = self.get_rp_api(session, f"launch/{launch_id}") launch_uuid = yaml_to_dict(response.text).get("uuid") - elif launch_uuid: - response = session.get( - url=f"{url}/launch/uuid/{launch_uuid}", - headers=headers) - self.handle_response(response) - launch_id = yaml_to_dict(response.text).get("id") + if launch_uuid and not launch_id: + response = self.get_rp_api(session, f"launch/uuid/{launch_uuid}") + launch_id = yaml_to_dict(response.text).get("id") + # Print the launch info + if not create_launch: launch_name = yaml_to_dict(response.text).get("name") - self.verbose("launch", launch_name, color="yellow") - launch_url = f"{endpoint}/ui/#{project}/launches/all/{launch_id}" + self.verbose("launch", launch_name, color="green") + self.verbose("id", launch_id, "yellow", shift=1) assert launch_uuid is not None self.verbose("uuid", launch_uuid, "yellow", shift=1) self.data.launch_uuid = launch_uuid + launch_url = f"{self.endpoint}/ui/#{self.project}/launches/all/{launch_id}" + if create_suite: # Create a suite suite_name = self.step.plan.name self.info("suite", suite_name, color="cyan") - response = session.post( - url=f"{url}/item", - headers=headers, - json={ - "name": suite_name, - "description": self.step.plan.summary, - "attributes": attributes, - "startTime": launch_time, - "launchUuid": launch_uuid, - "type": "suite"}) - self.handle_response(response) + response = self.post_rp_api(session, "item", + json={"name": suite_name, + "description": self.step.plan.summary, + "attributes": attributes, + "startTime": launch_time, + "launchUuid": launch_uuid, + "type": "suite"}) suite_uuid = yaml_to_dict(response.text).get("id") assert suite_uuid is not None + + elif suite_name: + self.info("suite", suite_name, color="green") + self.verbose("id", suite_id, "yellow", shift=1) + + if suite_uuid: self.verbose("uuid", suite_uuid, "yellow", shift=1) + self.data.suite_uuid = suite_uuid # For each test for test in self.step.plan.discover.tests(): - test_time = str(int(time() * 1000)) + test_time = self.time() if executed: result = next((result for result in self.step.plan.execute.results() if test.serial_number == result.serial_number), None) test_time = result.start_time - # TODO: for happz, connect Test to Result if possible # (but now it is probably too hackish to be fixed) @@ -369,25 +416,27 @@ def go(self) -> None: for key, value in test.environment.items() if not re.search(envar_pattern, key)] - # Create a test item - self.info("test", test.name, color="cyan") - response = session.post( - url=f"{url}/item{f'/{suite_uuid}' if suite_uuid else ''}", - headers=headers, - json={ - "name": test.name, - "description": test.summary, - "attributes": item_attributes, - "parameters": env_vars, - "codeRef": test.web_link() or None, - "launchUuid": launch_uuid, - "type": "step", - "testCaseId": test.id or None, - "startTime": test_time}) - self.handle_response(response) - item_uuid = yaml_to_dict(response.text).get("id") - assert item_uuid is not None - self.verbose("uuid", item_uuid, "yellow", shift=1) + if create_test: + # Create a test item + self.info("test", test.name, color="cyan") + response = self.post_rp_api(session, + f"item{f'/{suite_uuid}' if {suite_uuid} else ''}", + json={ + "name": test.name, + "description": test.summary, + "attributes": item_attributes, + "parameters": env_vars, + "codeRef": test.web_link() or None, + "launchUuid": launch_uuid, + "type": "step", + "testCaseId": test.id or None, + "startTime": test_time}) + item_uuid = yaml_to_dict(response.text).get("id") + assert item_uuid is not None + self.verbose("uuid", item_uuid, "yellow", shift=1) + self.data.test_uuids[test.serial_number] = item_uuid + else: + item_uuid = self.data.test_uuids[test.serial_number] # to supoort idle tests status = "SKIPPED" @@ -403,69 +452,65 @@ def go(self) -> None: status = self.TMT_TO_RP_RESULT_STATUS[result.result] # Upload log - response = session.post( - url=f"{url}/log/entry", - headers=headers, - json={ - "message": log, - "itemUuid": item_uuid, - "launchUuid": launch_uuid, - "level": level, - "time": result.end_time}) - self.handle_response(response) + response = self.post_rp_api(session, "log/entry", + json={"message": log, + "itemUuid": item_uuid, + "launchUuid": launch_uuid, + "level": level, + "time": result.end_time}) # Write out failures if index == 0 and status == "FAILED": message = result.failures(log) - response = session.post( - url=f"{url}/log/entry", - headers=headers, - json={ - "message": message, - "itemUuid": item_uuid, - "launchUuid": launch_uuid, - "level": "ERROR", - "time": result.end_time}) - self.handle_response(response) + response = self.post_rp_api(session, "log/entry", + json={"message": message, + "itemUuid": item_uuid, + "launchUuid": launch_uuid, + "level": "ERROR", + "time": result.end_time}) # TODO: Add tmt files as attachments test_time = result.end_time # Finish the test item - response = session.put( - url=f"{url}/item/{item_uuid}", - headers=headers, + # # response = self.put_rp_api(session, f"item/{item_uuid}", + # # json={"launchUuid": launch_uuid, + # # "endTime": test_time, + # # "status": "PASSED"}) + response = self.put_rp_api( + session, + f"item/{item_uuid}", json={ "launchUuid": launch_uuid, "endTime": test_time, "status": status, - "issue": {"issueType": dt_locator or "ti001"}}) + "issue": { + "issueType": self.get_defect_type_locator( + session, + defect_type)}}) self.handle_response(response) launch_time = test_time + # TODO: resolve problem with reporting original defect type (idle) + # after additional report of results + # - temporary solution idea: + # if again_additional_tests and status failed, + # get test_id, report passed and then again failed + if create_suite: # Finish the test suite - response = session.put( - url=f"{url}/item/{suite_uuid}", - headers=headers, - json={ - "launchUuid": launch_uuid, - "endTime": launch_time}) - self.handle_response(response) + response = self.put_rp_api(session, f"item/{suite_uuid}", + json={"launchUuid": launch_uuid, + "endTime": launch_time}) is_the_last_plan = self.step.plan == self.step.plan.my_run.plans[-1] - additional_upload = self.get("upload_to_launch") is not None \ - or self.get("upload_to_suite") is not None if ((launch_per_plan or (suite_per_plan and is_the_last_plan)) and not additional_upload): # Finish the launch - response = session.put( - url=f"{url}/launch/{launch_uuid}/finish", - headers=headers, - json={"endTime": launch_time}) - self.handle_response(response) + response = self.put_rp_api(session, f"launch/{launch_uuid}/finish", + json={"endTime": launch_time}) launch_url = yaml_to_dict(response.text).get("link") assert launch_url is not None From b2be09419b762e36ea2aa91a87ef901b6d1d229a Mon Sep 17 00:00:00 2001 From: nbubakov Date: Fri, 12 Jan 2024 20:49:33 +0100 Subject: [PATCH 7/7] Updated ReportPortal plugin extension based on review feedbacks + resolved mypy conflicts --- spec/plans/report.fmf | 40 +++- tmt/schemas/report/reportportal.yaml | 24 ++- tmt/steps/report/reportportal.py | 293 ++++++++++++++------------- 3 files changed, 200 insertions(+), 157 deletions(-) diff --git a/spec/plans/report.fmf b/spec/plans/report.fmf index ba9b78b9ca..7d52ce94b2 100644 --- a/spec/plans/report.fmf +++ b/spec/plans/report.fmf @@ -104,21 +104,47 @@ description: web page, filter them via context attributes and get links to detailed test output and other test information. description: - Fill json with test results and other fmf data per each plan, + Provide test results and fmf data per each plan, and send it to a Report Portal instance via its API with token, url and project name given. example: - | - # Set environment variables accoriding to TMT_PLUGIN_REPORT_REPORTPORTAL_${OPTION} + # Optionally set environment variables according to TMT_PLUGIN_REPORT_REPORTPORTAL_${OPTION} export TMT_PLUGIN_REPORT_REPORTPORTAL_URL=${url-to-RP-instance} export TMT_PLUGIN_REPORT_REPORTPORTAL_TOKEN=${token-from-RP-profile} - | - # Enable ReportPortal report from the command line + # Enable ReportPortal report from the command line depending on the use case: + + ## Simple upload with all project, url endpoint and user token passed in command line + tmt run --all report --how reportportal --project=baseosqe --url="https://reportportal.xxx.com" --token="abc...789" + + ## Simple upload with url and token exported in environment variable tmt run --all report --how reportportal --project=baseosqe - tmt run --all report --how reportportal --project=baseosqe --launch=test_plan - tmt run --all report --how reportportal --project=baseosqe --url=... --token=... - tmt run --all report --how reportportal --project=baseosqe --exclude-variables="^(TMT|PACKIT|TESTING_FARM).*" - tmt run -a report -h reportportal --merge --launch "Errata" --attributes "errata:123456" + + ## Upload with project name in fmf data, filtering out parameters (environemnt variables) that tend to be unique and break the history aggregation + tmt run --all report --how reportportal --exclude-variables="^(TMT|PACKIT|TESTING_FARM).*" + + ## Upload all plans as suites into one ReportPortal launch + tmt run --all report --how reportportal --suite-per-plan --launch=Errata --launch-description="..." + + ## Rerun the launch with suite structure for the test results to be uploaded into the latest launch with the same name as a new 'Retry' tab (mapping based on unique paths) + tmt run --all report --how reportportal --suite-per-plan --launch=Errata --launch-rerun + + ## Rerun the tmt run and append the new result logs under the previous one uploaded in ReportPortal (precise mapping) + tmt run --id run-012 --all report --how reportportal --again + + ## Additional upload of new suites into given launch with suite structure + tmt run --all report --how reportportal --suite-per-plan --upload-to-launch=4321 + + ## Additional upload of new tests into given launch with non-suite structure + tmt run --all report --how reportportal --launch-per-plan --upload-to-launch=1234 + + ## Additional upload of new tests into given suite + tmt run --all report --how reportportal --upload-to-suite=123456 + + ## Upload Idle tests, then execute it and add result logs into prepared empty tests + tmt run discover report --how reportportal --defect-type=Idle + tmt run --last --all report --how reportportal --again - | # Use ReportPortal as the default report for given plan report: diff --git a/tmt/schemas/report/reportportal.yaml b/tmt/schemas/report/reportportal.yaml index ac3517e1db..b901228fa8 100644 --- a/tmt/schemas/report/reportportal.yaml +++ b/tmt/schemas/report/reportportal.yaml @@ -34,10 +34,26 @@ properties: launch: type: string - attributes: - type: array - items: - type: string + launch-description: + type: string + + launch-per-plan: + type: boolean + + suite-per-plan: + type: boolean + + upload-to-launch: + type: string + + upload-to-suite: + type: string + + launch-rerun: + type: boolean + + defect_type: + type: string exclude-variables: type: string diff --git a/tmt/steps/report/reportportal.py b/tmt/steps/report/reportportal.py index bf228437a7..557b2b96b8 100644 --- a/tmt/steps/report/reportportal.py +++ b/tmt/steps/report/reportportal.py @@ -34,7 +34,7 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): metavar="LAUNCH_NAME", default=os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_LAUNCH'), help="The launch name (name of plan per launch is used by default).") - launch_description: str = field( + launch_description: Optional[str] = field( option="--launch-description", metavar="LAUNCH_DESCRIPTION", default=os.getenv('TMT_PLUGIN_REPORT_REPORTPORTAL_LAUNCH_DESCRIPTION'), @@ -52,13 +52,13 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): help="Mapping suite per plan, creating one launch and continous uploading suites into it. " "Recommended to use with '--launch' and '--launch-description' options." "Can be used with '--upload-to-launch' option to avoid creating a new launch.") - upload_to_launch: str = field( + upload_to_launch: Optional[str] = field( option="--upload-to-launch", metavar="LAUNCH_ID", default=None, help="Pass the launch ID for an additional test/suite upload to an existing launch. " "ID can be found in the launch URL.") - upload_to_suite: str = field( + upload_to_suite: Optional[str] = field( option="--upload-to-suite", metavar="LAUNCH_SUITE", default=None, @@ -68,7 +68,7 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): option="--launch-rerun", default=False, is_flag=True, - help="Rerun the launch based on unique test paths and ids to create Retry item" + help="Rerun the last launch based on its name and unique test paths to create Retry item" "with a new version per each test. Supported in 'suite-per-plan' structure only.") defect_type: Optional[str] = field( option="--defect-type", @@ -76,8 +76,6 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): default=None, help="Pass the defect type to be used for failed test, which is defined in the project" " (e.g. 'Idle'). 'To Investigate' is used by default.") - # TODO: test how to create empty test skeleton, all with Idle defect_type - # (as it reports defect_type only when it fails) exclude_variables: str = field( option="--exclude-variables", metavar="PATTERN", @@ -88,16 +86,16 @@ class ReportReportPortalData(tmt.steps.report.ReportStepData): "to prevent overloading and to preserve the history aggregation" "for ReportPortal item if tmt id is not provided") - launch_url: str = "" - launch_uuid: str = "" - suite_uuid: str = "" + launch_url: Optional[str] = None + launch_uuid: Optional[str] = None + suite_uuid: Optional[str] = None test_uuids: dict[int, str] = field( default_factory=dict ) @tmt.steps.provides_method("reportportal") -class ReportReportPortal(tmt.steps.report.ReportPlugin): +class ReportReportPortal(tmt.steps.report.ReportPlugin[ReportReportPortalData]): """ Report test results to a ReportPortal instance via API. @@ -110,7 +108,7 @@ class ReportReportPortal(tmt.steps.report.ReportPlugin): * optional API version to override the default one (v1) * optional launch name to override the deafult name based on the tmt plan name - + In addition to command line options it's possible to use environment variables: @@ -174,25 +172,44 @@ def handle_response(self, response: requests.Response) -> None: raise tmt.utils.ReportError( f"Received non-ok status code from ReportPortal: {response.text}") - self.debug("Response code from the endpoint", str(response.status_code)) - self.debug("Message from the endpoint", str(response.text)) + self.debug("Response code from the endpoint", response.status_code) + self.debug("Message from the endpoint", response.text) + + def check_options(self) -> None: + """ + Write warning if there might be caused an unexpected behaviour by the option combinations + """ + # TODO: Update restriction of forbiden option combinations based on feedback. + + if self.data.launch_per_plan and self.data.suite_per_plan: + raise tmt.utils.ReportError( + "The options '--launch-per-plan' and '--suite-per-plan' are mutually exclusive. " + "Choose one of them only.") + + if self.data.launch_rerun and (self.data.upload_to_launch or self.data.upload_to_suite): + self.warn("Unexpected option combination: " + "'--launch-rerun' is ignored when uploading additional tests.") + + if not self.data.suite_per_plan and self.data.launch_rerun: + self.warn("Unexpected option combination: '--launch-rerun' " + "may cause an unexpected behaviour with launch-per-plan structure") def time(self) -> str: return str(int(time() * 1000)) def get_headers(self) -> dict[str, str]: - return {"Authorization": "bearer " + self.token, + return {"Authorization": "bearer " + str(self.data.token), "accept": "*/*", "Content-Type": "application/json"} def get_url(self) -> str: api_version = os.getenv( 'TMT_PLUGIN_REPORT_REPORTPORTAL_API_VERSION') or self.DEFAULT_API_VERSION - return f"{self.endpoint}/api/{api_version}/{self.project}" + return f"{self.data.url}/api/{api_version}/{self.data.project}" def construct_launch_attributes(self, suite_per_plan: bool, - attributes: dict[str, str]) -> dict[str, str]: - if not suite_per_plan: + attributes: list[dict[str, str]]) -> list[dict[str, str]]: + if not suite_per_plan or not self.step.plan.my_run: return attributes.copy() # Get common attributes across the plans @@ -207,44 +224,30 @@ def construct_launch_attributes(self, suite_per_plan: bool, result_dict = tmp_dict return [{'key': key, 'value': value} for key, value in result_dict.items()] - def get_defect_type_locator(self, session: requests.Session, defect_type: str) -> str: + def get_defect_type_locator(self, session: requests.Session, + defect_type: Optional[str]) -> str: if not defect_type: return "ti001" - # Get defect type locator via api response = self.get_rp_api(session, "settings") defect_types = yaml_to_dict(response.text).get("subTypes") + if not defect_types: + return "ti001" dt_tmp = [dt['locator'] for dt in defect_types['TO_INVESTIGATE'] if dt['longName'].lower() == defect_type.lower()] dt_locator = dt_tmp[0] if dt_tmp else None if not dt_locator: - raise tmt.utils.ReportError( - f"Defect type '{defect_type}' is not be defined in the project {self.project}") + raise tmt.utils.ReportError(f"Defect type '{defect_type}' " + "is not be defined in the project {self.data.project}") self.verbose("defect_typ", defect_type, "yellow") - return dt_locator + return str(dt_locator) - def get_rp_api(self, session: requests.Session, data_path: str) -> str: + def get_rp_api(self, session: requests.Session, data_path: str) -> requests.Response: response = session.get(url=f"{self.get_url()}/{data_path}", headers=self.get_headers()) self.handle_response(response) return response - def post_rp_api(self, session: requests.Session, item_path: str, json: dict[str, str]) -> str: - response = session.post( - url=f"{self.get_url()}/{item_path}", - headers=self.get_headers(), - json=json) - self.handle_response(response) - return response - - def put_rp_api(self, session: requests.Session, item_path: str, json: dict[str, str]) -> str: - response = session.put( - url=f"{self.get_url()}/{item_path}", - headers=self.get_headers(), - json=json) - self.handle_response(response) - return response - def go(self) -> None: """ Report test results to the endpoint @@ -253,76 +256,60 @@ def go(self) -> None: fill it with all parts needed and report the logs. """ - # TODO: (to be deleted after review) - # * resolve the problem with mypy - # * check the problem with --upload-to-suite functonality - # * check the param per plan, and do the matching when uploading (launch_uuid) - # * upload documentation (help and spec) - # * add the tests for new features --> another PR - # * add uploading files --> another PR - # * edit schemas - # * check optionality in arguments - # * restrict combinations + set warning - # - forbid rerun && launch-per-plan - # - forbid upload-to-suite && launch-per-plan - # * try read a launch_uuid at first plan (self.step.plan.report.launch_uuid) - # * rewrite into smarter and neater code (bolean logic for option combinations) - super().go() - self.endpoint = self.get("url") - if not self.endpoint: + if not self.data.url: raise tmt.utils.ReportError("No ReportPortal endpoint url provided.") - self.endpoint = self.endpoint.rstrip("/") + self.data.url = self.data.url.rstrip("/") - self.project = self.get("project") - if not self.project: + if not self.data.project: raise tmt.utils.ReportError("No ReportPortal project provided.") - self.token = self.get("token") - if not self.token: + if not self.data.token: raise tmt.utils.ReportError("No ReportPortal token provided.") + if not self.step.plan.my_run: + raise tmt.utils.ReportError("No run data available.") + + self.check_options() + launch_time = self.time() - # Supporting idle tests - executed = False - if len(self.step.plan.execute.results()) > 0: - launch_time = self.step.plan.execute.results()[0].start_time - executed = True + # Support for idle tests + executed = bool(self.step.plan.execute.results()) + if executed: + launch_time = self.step.plan.execute.results()[0].start_time or self.time() # Create launch, suites (if "--suite_per_plan") and tests; # or report to existing launch/suite if its id is given - suite_per_plan = self.get("suite_per_plan") - launch_per_plan = self.get("launch_per_plan") + suite_per_plan = self.data.suite_per_plan + launch_per_plan = self.data.launch_per_plan if not launch_per_plan and not suite_per_plan: launch_per_plan = True # by default - elif launch_per_plan and suite_per_plan: - raise tmt.utils.ReportError( - "The options '--launch-per-plan' and " - "'--suite-per-plan' are mutually exclusive. Choose one of them only.") - suite_id = self.get("upload_to_suite") - launch_id = self.get("upload_to_launch") + suite_id = self.data.upload_to_suite + launch_id = self.data.upload_to_launch - suite_uuid = self.get("suite_uuid") - launch_uuid = self.get("launch_uuid") + suite_uuid = self.data.suite_uuid + launch_uuid = self.data.launch_uuid additional_upload = suite_id or launch_id or launch_uuid is_the_first_plan = self.step.plan == self.step.plan.my_run.plans[0] if not launch_uuid and suite_per_plan and not is_the_first_plan: - launch_uuid = self.step.plan.my_run.plans[0].report.data[0].launch_uuid + rp_phases = list(self.step.plan.my_run.plans[0].report.phases(ReportReportPortal)) + if rp_phases: + launch_uuid = rp_phases[0].data.launch_uuid create_test = not self.data.test_uuids create_suite = suite_per_plan and not (suite_uuid or suite_id) create_launch = not (launch_uuid or launch_id or suite_uuid or suite_id) - launch_name = self.get("launch") or self.step.plan.name + launch_name = self.data.launch or self.step.plan.name suite_name = "" launch_url = "" - launch_rerun = self.get("launch_rerun") - envar_pattern = self.get("exclude-variables") or "$^" - defect_type = self.get("defect_type") + launch_rerun = self.data.launch_rerun + envar_pattern = self.data.exclude_variables or "$^" + defect_type = self.data.defect_type attributes = [ {'key': key, 'value': value[0]} @@ -330,7 +317,7 @@ def go(self) -> None: launch_attributes = self.construct_launch_attributes(suite_per_plan, attributes) - launch_description = self.get("launch_description") or self.step.plan.summary + launch_description = self.data.launch_description or self.step.plan.summary # Communication with RP instance with tmt.utils.retry_session() as session: @@ -339,12 +326,15 @@ def go(self) -> None: # Create a launch self.info("launch", launch_name, color="cyan") - response = self.post_rp_api(session, "launch", - json={"name": launch_name, - "description": launch_description, - "attributes": launch_attributes, - "startTime": launch_time, - "rerun": launch_rerun}) + response = session.post( + url=f"{self.get_url()}/launch", + headers=self.get_headers(), + json={"name": launch_name, + "description": launch_description, + "attributes": launch_attributes, + "startTime": launch_time, + "rerun": launch_rerun}) + self.handle_response(response) launch_uuid = yaml_to_dict(response.text).get("id") else: @@ -352,8 +342,7 @@ def go(self) -> None: if suite_id: response = self.get_rp_api(session, f"item/{suite_id}") suite_uuid = yaml_to_dict(response.text).get("uuid") - # self.info("suite_id", suite_id, color="yellow") - suite_name = yaml_to_dict(response.text).get("name") + suite_name = str(yaml_to_dict(response.text).get("name")) launch_id = yaml_to_dict(response.text).get("launchId") if launch_id: @@ -366,7 +355,7 @@ def go(self) -> None: # Print the launch info if not create_launch: - launch_name = yaml_to_dict(response.text).get("name") + launch_name = yaml_to_dict(response.text).get("name") or "" self.verbose("launch", launch_name, color="green") self.verbose("id", launch_id, "yellow", shift=1) @@ -374,19 +363,22 @@ def go(self) -> None: self.verbose("uuid", launch_uuid, "yellow", shift=1) self.data.launch_uuid = launch_uuid - launch_url = f"{self.endpoint}/ui/#{self.project}/launches/all/{launch_id}" + launch_url = f"{self.data.url}/ui/#{self.data.project}/launches/all/{launch_id}" if create_suite: # Create a suite suite_name = self.step.plan.name self.info("suite", suite_name, color="cyan") - response = self.post_rp_api(session, "item", - json={"name": suite_name, - "description": self.step.plan.summary, - "attributes": attributes, - "startTime": launch_time, - "launchUuid": launch_uuid, - "type": "suite"}) + response = session.post( + url=f"{self.get_url()}/item", + headers=self.get_headers(), + json={"name": suite_name, + "description": self.step.plan.summary, + "attributes": attributes, + "startTime": launch_time, + "launchUuid": launch_uuid, + "type": "suite"}) + self.handle_response(response) suite_uuid = yaml_to_dict(response.text).get("id") assert suite_uuid is not None @@ -404,7 +396,8 @@ def go(self) -> None: if executed: result = next((result for result in self.step.plan.execute.results() if test.serial_number == result.serial_number), None) - test_time = result.start_time + if result: + test_time = result.start_time or self.time() # TODO: for happz, connect Test to Result if possible # (but now it is probably too hackish to be fixed) @@ -419,18 +412,19 @@ def go(self) -> None: if create_test: # Create a test item self.info("test", test.name, color="cyan") - response = self.post_rp_api(session, - f"item{f'/{suite_uuid}' if {suite_uuid} else ''}", - json={ - "name": test.name, - "description": test.summary, - "attributes": item_attributes, - "parameters": env_vars, - "codeRef": test.web_link() or None, - "launchUuid": launch_uuid, - "type": "step", - "testCaseId": test.id or None, - "startTime": test_time}) + response = session.post( + url=f"{self.get_url()}/item{f'/{suite_uuid}' if suite_uuid else ''}", + headers=self.get_headers(), + json={"name": test.name, + "description": test.summary, + "attributes": item_attributes, + "parameters": env_vars, + "codeRef": test.web_link() or None, + "launchUuid": launch_uuid, + "type": "step", + "testCaseId": test.id or None, + "startTime": test_time}) + self.handle_response(response) item_uuid = yaml_to_dict(response.text).get("id") assert item_uuid is not None self.verbose("uuid", item_uuid, "yellow", shift=1) @@ -438,9 +432,9 @@ def go(self) -> None: else: item_uuid = self.data.test_uuids[test.serial_number] - # to supoort idle tests + # Support for idle tests status = "SKIPPED" - if executed: + if executed and result: # For each log for index, log_path in enumerate(result.log): try: @@ -452,66 +446,73 @@ def go(self) -> None: status = self.TMT_TO_RP_RESULT_STATUS[result.result] # Upload log - response = self.post_rp_api(session, "log/entry", - json={"message": log, - "itemUuid": item_uuid, - "launchUuid": launch_uuid, - "level": level, - "time": result.end_time}) + response = session.post( + url=f"{self.get_url()}/log/entry", + headers=self.get_headers(), + json={"message": log, + "itemUuid": item_uuid, + "launchUuid": launch_uuid, + "level": level, + "time": result.end_time}) + self.handle_response(response) # Write out failures if index == 0 and status == "FAILED": message = result.failures(log) - response = self.post_rp_api(session, "log/entry", - json={"message": message, - "itemUuid": item_uuid, - "launchUuid": launch_uuid, - "level": "ERROR", - "time": result.end_time}) + response = session.post( + url=f"{self.get_url()}/log/entry", + headers=self.get_headers(), + json={"message": message, + "itemUuid": item_uuid, + "launchUuid": launch_uuid, + "level": "ERROR", + "time": result.end_time}) + self.handle_response(response) # TODO: Add tmt files as attachments - test_time = result.end_time + test_time = result.end_time or self.time() # Finish the test item - # # response = self.put_rp_api(session, f"item/{item_uuid}", - # # json={"launchUuid": launch_uuid, - # # "endTime": test_time, - # # "status": "PASSED"}) - response = self.put_rp_api( - session, - f"item/{item_uuid}", + response = session.put( + url=f"{self.get_url()}/item/{item_uuid}", + headers=self.get_headers(), json={ "launchUuid": launch_uuid, "endTime": test_time, "status": status, "issue": { - "issueType": self.get_defect_type_locator( - session, - defect_type)}}) + "issueType": self.get_defect_type_locator(session, defect_type)}}) self.handle_response(response) launch_time = test_time - # TODO: resolve problem with reporting original defect type (idle) - # after additional report of results - # - temporary solution idea: + # TODO: Resolve the problem with reporting original defect type (idle) + # after additional report of results + # Temporary solution idea: # if again_additional_tests and status failed, # get test_id, report passed and then again failed if create_suite: # Finish the test suite - response = self.put_rp_api(session, f"item/{suite_uuid}", - json={"launchUuid": launch_uuid, - "endTime": launch_time}) + response = session.put( + url=f"{self.get_url()}/item{f'/{suite_uuid}' if suite_uuid else ''}", + headers=self.get_headers(), + json={ + "launchUuid": launch_uuid, + "endTime": launch_time}) + self.handle_response(response) is_the_last_plan = self.step.plan == self.step.plan.my_run.plans[-1] if ((launch_per_plan or (suite_per_plan and is_the_last_plan)) and not additional_upload): # Finish the launch - response = self.put_rp_api(session, f"launch/{launch_uuid}/finish", - json={"endTime": launch_time}) - launch_url = yaml_to_dict(response.text).get("link") + response = session.put( + url=f"{self.get_url()}/launch/{launch_uuid}/finish", + headers=self.get_headers(), + json={"endTime": launch_time}) + self.handle_response(response) + launch_url = str(yaml_to_dict(response.text).get("link")) assert launch_url is not None self.info("url", launch_url, "magenta")