Generate XUnit Polarion flavor using JUnit report plugin

tests/report/polarion/test.sh

@@ -11,9 +11,9 @@ rlJournalStart
  rlRun "tmt run -avr execute report -h polarion --project-id RHELBASEOS --no-upload --planned-in RHEL-9.1.0 --file xunit.xml 2>&1 >/dev/null | tee output" 2
  rlAssertGrep "1 test passed, 1 test failed and 1 error" "output"
  rlAssertGrep '<testsuite name="/plan" disabled="0" errors="1" failures="1" skipped="0" tests="3"' "xunit.xml"
- rlAssertGrep '<property name="polarion-project-id" value="RHELBASEOS" />' "xunit.xml"
- rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10914" />' "xunit.xml"
- rlAssertGrep '<property name="polarion-custom-plannedin" value="RHEL-9.1.0" />' "xunit.xml"
+ rlAssertGrep '<property name="polarion-project-id" value="RHELBASEOS"/>' "xunit.xml"
+ rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10914"/>' "xunit.xml"
+ rlAssertGrep '<property name="polarion-custom-plannedin" value="RHEL-9.1.0"/>' "xunit.xml"
  rlAssertGrep "Maximum test time '2s' exceeded." "xunit.xml"
  rlPhaseEnd
 

tmt/steps/report/junit.py

@@ -295,7 +295,7 @@ class ReportJUnitData(tmt.steps.report.ReportStepData):
  default=DEFAULT_FLAVOR_NAME,
  option='--flavor',
  metavar='FLAVOR',
- choices=[DEFAULT_FLAVOR_NAME, CUSTOM_FLAVOR_NAME],
+ choices=[DEFAULT_FLAVOR_NAME, CUSTOM_FLAVOR_NAME, 'polarion'],
  help=f"Name of a JUnit flavor to generate. By default, the '{DEFAULT_FLAVOR_NAME}' flavor "
  "is used.")
 

tmt/steps/report/junit/schemas/polarion.xsd

@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<!--
+ This schema supports only a subset of the features provided by the
+ `xml-junit` library. Additionally, many attributes are explicitly set as
+ required. This is intentional to limit the currently supported features of
+ the TMT Polarion report plugin .
+
+ The Polarion `xunit.xml` is almost the same as default output of junit
+ report plugin but it must allow definition of `properties` inside of
+ `testsuites` (NOT `testsuite`) and `testcase`.
+-->
+
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+
+ <xs:element name="failure">
+ <xs:complexType mixed="true">
+ <xs:attribute name="type" type="xs:string" use="required"/>
+ <xs:attribute name="message" type="xs:string" use="required"/>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="error">
+ <xs:complexType mixed="true">
+ <xs:attribute name="type" type="xs:string" use="required"/>
+ <xs:attribute name="message" type="xs:string" use="required"/>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="skipped">
+ <xs:complexType mixed="true">
+ <xs:attribute name="type" type="xs:string" use="required"/>
+ <xs:attribute name="message" type="xs:string" use="required"/>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="system-err" type="xs:string"/>
+ <xs:element name="system-out" type="xs:string"/>
+
+ <xs:element name="properties">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="property" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="property">
+ <xs:complexType>
+ <xs:attribute name="name" type="xs:string" use="required"/>
+ <xs:attribute name="value" type="xs:string" use="required"/>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="testcase">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="skipped" minOccurs="0" maxOccurs="1"/>
+ <xs:element ref="error" minOccurs="0" maxOccurs="1"/>
+ <xs:element ref="failure" minOccurs="0" maxOccurs="1"/>
+ <xs:element ref="system-out" minOccurs="0" maxOccurs="1"/>
+ <xs:element ref="system-err" minOccurs="0" maxOccurs="1"/>
+ <xs:element ref="properties" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="xs:string" use="required"/>
+ <xs:attribute name="time" type="xs:string" use="optional"/>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="testsuite">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="testcase" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ <xs:attribute name="name" type="xs:string" use="required"/>
+ <xs:attribute name="tests" type="xs:string" use="required"/>
+ <xs:attribute name="failures" type="xs:string" use="required"/>
+ <xs:attribute name="errors" type="xs:string" use="required"/>
+ <xs:attribute name="disabled" type="xs:string" use="required"/>
+ <xs:attribute name="skipped" type="xs:string" use="required"/>
+ <xs:attribute name="time" type="xs:string" use="required"/>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="testsuites">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="testsuite" minOccurs="1" maxOccurs="1"/>
+ <xs:element ref="properties" minOccurs="0" maxOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute name="time" type="xs:string" use="required"/>
+ <xs:attribute name="tests" type="xs:string" use="optional"/>
+ <xs:attribute name="failures" type="xs:string" use="optional"/>
+ <xs:attribute name="disabled" type="xs:string" use="optional"/>
+ <xs:attribute name="errors" type="xs:string" use="optional"/>
+ </xs:complexType>
+ </xs:element>
+</xs:schema>

tmt/steps/report/junit/templates/polarion.xml.j2

@@ -0,0 +1,10 @@
+{% extends "_base.xml.j2" %}
+
+{% block testsuites %}
+ {{ super() }}
+
+ {# Polarion XUnit must include the properties section in testsuites tag #}
+ {% with properties=TESTSUITES_PROPERTIES %}
+ {% include "includes/_properties.xml.j2" %}
+ {% endwith %}
+{% endblock %}

tmt/steps/report/polarion.py

@@ -2,16 +2,16 @@
 import datetime
 import os
 from typing import Optional
-from xml.etree import ElementTree
 
 from requests import post
 
 import tmt
 import tmt.steps
 import tmt.steps.report
+import tmt.utils
 from tmt.utils import Path, field
 
-from .junit import make_junit_xml
+from .junit import ResultsContext, make_junit_xml
 
 DEFAULT_NAME = 'xunit.xml'
 
@@ -23,7 +23,7 @@ class ReportPolarionData(tmt.steps.report.ReportStepData):
  option='--file',
  metavar='FILE',
  help='Path to the file to store xUnit in.',
- normalize=lambda key_address, raw_value, logger: Path(raw_value) if raw_value else None)
+ normalize=tmt.utils.normalize_path)
 
  upload: bool = field(
  default=True,
@@ -180,6 +180,13 @@ class ReportPolarionData(tmt.steps.report.ReportStepData):
  help='FIPS mode enabled or disabled for this run.'
  )
 
+ prettify: bool = field(
+ default=True,
+ option=('--prettify / --no-prettify'),
+ is_flag=True,
+ show_default=True,
+ help="Enable the XML pretty print for generated XUnit file.")
+
  include_output_log: bool = field(
  default=True,
  option=('--include-output-log / --no-include-output-log'),
@@ -213,6 +220,7 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None:
  self.step.plan.name.rsplit('/', 1)[1] + '_' +
  # Polarion server running with UTC timezone
  datetime.datetime.now(tz=datetime.timezone.utc).strftime("%Y%m%d%H%M%S"))
+
  title = title.replace('-', '_')
  project_id = self.data.project_id or os.getenv('TMT_PLUGIN_REPORT_POLARION_PROJECT_ID')
  template = self.data.template or os.getenv('TMT_PLUGIN_REPORT_POLARION_TEMPLATE')
@@ -224,53 +232,46 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None:
  'description', 'planned_in', 'assignee', 'pool_team', 'arch', 'platform', 'build',
  'sample_image', 'logs', 'compose_id', 'fips']
 
- xml_data = make_junit_xml(
- phase=self,
+ # TODO: The current behavior is that the None value is propagated into the final XML as a
+ # string 'None'. I'm not sure if this is expected behavior, but the code is there just to
+ # keep to compatibility with existing deployments. The 'None' strings appear in
+ # `polarion-project-id` and `polarion-project-span-ids` property attributes.
+ if project_id is None:
+ project_id = str(project_id)
 
- # TODO: Explicitly use 'default' flavor until the 'polarion' flavor
- # gets implemented in junit report plugin.
- # flavor='polarion',
- flavor='default',
+ testsuites_properties = {}
 
- include_output_log=self.data.include_output_log)
-
- # S314: Any potential xml parser vulnerability mitigation would require defusedxml package
- xml_tree = ElementTree.fromstring(xml_data) # noqa: S314
- properties = {
- 'polarion-project-id': project_id,
- 'polarion-user-id': PolarionWorkItem._session.user_id,
- 'polarion-testrun-title': title,
- 'polarion-project-span-ids': project_id}
  for tr_field in other_testrun_fields:
  param = self.get(tr_field, os.getenv(f'TMT_PLUGIN_REPORT_POLARION_{tr_field.upper()}'))
  # TODO: remove the os.getenv when envvars in click work with steps in plans as well
  # as with steps on cmdline
  if param:
- properties[f"polarion-custom-{tr_field.replace('_', '')}"] = param
+ testsuites_properties[f"polarion-custom-{tr_field.replace('_', '')}"] = param
+
  if use_facts:
- properties['polarion-custom-hostname'] = \
- self.step.plan.provision.guests()[0].primary_address
- properties['polarion-custom-arch'] = self.step.plan.provision.guests()[0].facts.arch
+ testsuites_properties['polarion-custom-hostname'] = self.step.plan.provision.guests()[
+ 0].primary_address
+ testsuites_properties['polarion-custom-arch'] = self.step.plan.provision.guests()[
+ 0].facts.arch
+
  if template:
- properties['polarion-testrun-template-id'] = template
+ testsuites_properties['polarion-testrun-template-id'] = template
+
  logs = os.getenv('TMT_REPORT_ARTIFACTS_URL')
- if logs and 'polarion-custom-logs' not in properties:
- properties['polarion-custom-logs'] = logs
- testsuites_properties = ElementTree.SubElement(xml_tree, 'properties')
- for name, value in properties.items():
- ElementTree.SubElement(testsuites_properties, 'property', attrib={
- 'name': name, 'value': str(value)})
-
- testsuite = xml_tree.find('testsuite')
- project_span_ids = xml_tree.find(
- '*property[@name="polarion-project-span-ids"]')
-
- for result in self.step.plan.execute.results():
+ if logs and 'polarion-custom-logs' not in testsuites_properties:
+ testsuites_properties['polarion-custom-logs'] = logs
+
+ project_span_ids: list[str] = []
+
+ results_context = ResultsContext(self.step.plan.execute.results())
+
+ for result in results_context:
  if not result.ids or not any(result.ids.values()):
  self.warn(
  f"Test Case '{result.name}' is not exported to Polarion, "
  "please run 'tmt tests export --how polarion' on it.")
  continue
+
  work_item_id, test_project_id = find_polarion_case_ids(result.ids)
 
  if test_project_id is None:
@@ -280,26 +281,41 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None:
  assert work_item_id is not None
  assert project_span_ids is not None
 
- if test_project_id not in project_span_ids.attrib['value']:
- project_span_ids.attrib['value'] += f',{test_project_id}'
+ if test_project_id not in project_span_ids:
+ project_span_ids.append(test_project_id)
 
- test_properties = {
+ testcase_properties = {
  'polarion-testcase-id': work_item_id,
- 'polarion-testcase-project-id': test_project_id}
+ 'polarion-testcase-project-id': test_project_id,
+ }
 
- assert testsuite is not None
- test_case = testsuite.find(f"*[@name='{result.name}']")
- assert test_case is not None
- properties_elem = ElementTree.SubElement(test_case, 'properties')
- for name, value in test_properties.items():
- ElementTree.SubElement(properties_elem, 'property', attrib={
- 'name': name, 'value': value})
+ result.set_properties(testcase_properties)
 
  assert self.workdir is not None
 
+ testsuites_properties.update({
+ 'polarion-project-id': project_id,
+ 'polarion-user-id': PolarionWorkItem._session.user_id,
+ 'polarion-testrun-title': title,
+ 'polarion-project-span-ids': ','.join([project_id, *project_span_ids])})
+
+ xml_data = make_junit_xml(
+ phase=self,
+ flavor='polarion',
+ prettify=self.data.prettify,
+ include_output_log=self.data.include_output_log,
+ results_context=results_context,
+ TESTSUITES_PROPERTIES=[{'name': name, 'value': value}
+ for name, value in testsuites_properties.items()],
+ )
+
  f_path = self.data.file or self.workdir / DEFAULT_NAME
- with open(f_path, 'wb') as fw:
- ElementTree.ElementTree(xml_tree).write(fw, xml_declaration=True)
+
+ try:
+ with open(f_path, 'w') as fw:
+ fw.write(xml_data)
+ except Exception as error:
+ raise tmt.utils.ReportError(f"Failed to write the output '{f_path}' ({error}).")
 
  if upload:
  server_url = str(PolarionWorkItem._session._server.url)
@@ -311,14 +327,17 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None:
  PolarionWorkItem._session.password)
 
  response = post(
- polarion_import_url, auth=auth,
- files={'file': ('xunit.xml', ElementTree.tostring(xml_tree))}, timeout=10
- )
+ polarion_import_url,
+ auth=auth,
+ files={
+ 'file': ('xunit.xml', xml_data),
+ },
+ timeout=10)
  self.info(
  f'Response code is {response.status_code} with text: {response.text}')
  else:
- self.info("Polarion upload can be done manually using command:")
+ self.info('Polarion upload can be done manually using command:')
  self.info(
- "curl -k -u <USER>:<PASSWORD> -X POST -F file=@<XUNIT_XML_FILE_PATH> "
- "<POLARION_URL>/polarion/import/xunit")
+ 'curl -k -u <USER>:<PASSWORD> -X POST -F file=@<XUNIT_XML_FILE_PATH> '
+ '<POLARION_URL>/polarion/import/xunit')
  self.info('xUnit file saved at', f_path, 'yellow')