Skip to content

Commit

Permalink
Generate XUnit Polarion flavor using JUnit report plugin
Browse files Browse the repository at this point in the history
  • Loading branch information
seberm committed Sep 16, 2024
1 parent 4b3394b commit 3075f79
Show file tree
Hide file tree
Showing 8 changed files with 393 additions and 82 deletions.
5 changes: 5 additions & 0 deletions docs/releases.rst
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@
tmt-1.37
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The :ref:`/plugins/report/polarion` report plugin now uses Jinja template to
generate the XUnit file. It doesn't do any extra modifications to the XML tree
using an ``ElementTree`` anymore. Also the schema is now validated against the
XSD.

The :ref:`/plugins/report/junit` report plugin now validates all the XML
flavors against their respective XSD schemas and tries to prettify the final
XML output. These functionalities are always disabled for ``custom`` flavors.
Expand Down
141 changes: 135 additions & 6 deletions tests/report/polarion/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,143 @@ rlJournalStart
rlRun "set -o pipefail"
rlPhaseEnd

rlPhaseStartTest
rlRun "tmt run -avr execute report -h polarion --project-id RHELBASEOS --no-upload --planned-in RHEL-9.1.0 --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlPhaseStartTest 'Test the properties gets propagated to testsuites correctly'
rlRun "tmt run -avr execute report -h polarion --no-upload --project-id RHELBASEOS --template mytemplate --planned-in RHEL-9.1.0 --arch x86_64 --description mydesc --assignee myassignee --pool-team mypoolteam --platform myplatform --build mybuild --sample-image mysampleimage --logs mylogslocation --compose-id mycomposeid --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlAssertGrep "1 test passed, 1 test failed and 1 error" "output"
rlAssertGrep '<testsuite name="/plan" disabled="0" errors="1" failures="1" skipped="0" tests="3"' "xunit.xml"
rlAssertGrep '<property name="polarion-project-id" value="RHELBASEOS" />' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10914" />' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-plannedin" value="RHEL-9.1.0" />' "xunit.xml"
rlAssertGrep "Maximum test time '2s' exceeded." "xunit.xml"

# testsuites and testsuite tag attributes
rlAssertGrep '<testsuites disabled="0" errors="1" failures="1" tests="3"' "xunit.xml"
rlAssertGrep '<testsuite name="/plan" disabled="0" errors="1" failures="1" skipped="0" tests="3"' "xunit.xml"
# Main testsuite properties
rlAssertGrep '<property name="polarion-project-id" value="RHELBASEOS"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-project-span-ids" value="RHELBASEOS,RHELBASEOS"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testrun-title" value="plan_[0-9]\+"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testrun-template-id" value="mytemplate"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-user-id" value="' "xunit.xml"

# Custom testsuite properties
rlAssertGrep '<property name="polarion-custom-description" value="mydesc"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-plannedin" value="RHEL-9.1.0"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-assignee" value="myassignee"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-poolteam" value="mypoolteam"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-arch" value="x86_64"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-platform" value="myplatform"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-build" value="mybuild"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-sampleimage" value="mysampleimage"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-logs" value="mylogslocation"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-composeid" value="mycomposeid"/>' "xunit.xml"

# The testcase properties
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10913"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10914"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10915"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-project-id" value="RHELBASEOS"/>' "xunit.xml"
rlPhaseEnd

rlPhaseStartTest 'Test the facts properties'
rlRun "tmt run -avr execute report -h polarion --no-upload --project-id RHELBASEOS --use-facts --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlAssertGrep '<property name="polarion-custom-hostname" value="' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-arch" value="' "xunit.xml"
rlPhaseEnd

rlPhaseStartTest 'The "None" string should never be in a property value'
rlRun "tmt run -avr execute report -h polarion --no-upload --project-id RHELBASEOS --template '' --planned-in '' --arch '' --description '' --assignee '' --pool-team '' --platform '' --build '' --sample-image '' --logs '' --compose-id '' --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlAssertNotGrep 'value="None"' "xunit.xml"

rlRun "export \
TMT_PLUGIN_REPORT_POLARION_PROJECT_ID= \
TMT_PLUGIN_REPORT_POLARION_TITLE= \
TMT_PLUGIN_REPORT_POLARION_DESCRIPTION= \
TMT_PLUGIN_REPORT_POLARION_TEMPLATE= \
TMT_PLUGIN_REPORT_POLARION_PLANNED_IN= \
TMT_PLUGIN_REPORT_POLARION_ASSIGNEE= \
TMT_PLUGIN_REPORT_POLARION_POOL_TEAM= \
TMT_PLUGIN_REPORT_POLARION_ARCH= \
TMT_PLUGIN_REPORT_POLARION_PLATFORM= \
TMT_PLUGIN_REPORT_POLARION_BUILD= \
TMT_PLUGIN_REPORT_POLARION_SAMPLE_IMAGE= \
TMT_PLUGIN_REPORT_POLARION_LOGS= \
TMT_PLUGIN_REPORT_POLARION_COMPOSE_ID= \
"
rlRun "tmt run -avr execute report -h polarion --no-upload --project-id RHELBASEOS --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlAssertNotGrep 'value="None"' "xunit.xml"
rlPhaseEnd

rlPhaseStartTest 'Check the plugin behavior based on setting ENV variables'
rlRun "export \
TMT_PLUGIN_REPORT_POLARION_PROJECT_ID=myprojectid \
TMT_PLUGIN_REPORT_POLARION_TITLE=mytitle \
TMT_PLUGIN_REPORT_POLARION_DESCRIPTION=mydesc \
TMT_PLUGIN_REPORT_POLARION_TEMPLATE=mytemplate \
TMT_PLUGIN_REPORT_POLARION_PLANNED_IN=myplannedin \
TMT_PLUGIN_REPORT_POLARION_ASSIGNEE=myassignee \
TMT_PLUGIN_REPORT_POLARION_POOL_TEAM=mypoolteam \
TMT_PLUGIN_REPORT_POLARION_ARCH=x86_64 \
TMT_PLUGIN_REPORT_POLARION_PLATFORM=myplatform \
TMT_PLUGIN_REPORT_POLARION_BUILD=mybuild \
TMT_PLUGIN_REPORT_POLARION_SAMPLE_IMAGE=mysampleimage \
TMT_PLUGIN_REPORT_POLARION_LOGS=mylogslocation \
TMT_PLUGIN_REPORT_POLARION_COMPOSE_ID=mycomposeid \
"

rlRun "tmt run -avr execute report -h polarion --no-upload --file xunit.xml 2>&1 >/dev/null | tee output" 2
# Main testsuite properties
rlAssertGrep '<property name="polarion-project-id" value="myprojectid"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-project-span-ids" value="myprojectid,RHELBASEOS"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testrun-title" value="mytitle"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testrun-template-id" value="mytemplate"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-user-id" value="' "xunit.xml"

# Custom testsuite properties
rlAssertGrep '<property name="polarion-custom-description" value="mydesc"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-plannedin" value="myplannedin"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-assignee" value="myassignee"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-poolteam" value="mypoolteam"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-arch" value="x86_64"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-platform" value="myplatform"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-build" value="mybuild"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-sampleimage" value="mysampleimage"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-logs" value="mylogslocation"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-composeid" value="mycomposeid"/>' "xunit.xml"

# The testcase properties
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10913"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10914"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-id" value="BASEOS-10915"/>' "xunit.xml"
rlAssertGrep '<property name="polarion-testcase-project-id" value="RHELBASEOS"/>' "xunit.xml"
rlPhaseEnd

rlPhaseStartTest 'Check the plugin behavior based on TMT_PLUGIN_REPORT_POLARION_USE_FACTS env variable'
# Make sure all ENV variables are unset
rlRun "unset \
TMT_PLUGIN_REPORT_POLARION_PROJECT_ID \
TMT_PLUGIN_REPORT_POLARION_TITLE \
TMT_PLUGIN_REPORT_POLARION_DESCRIPTION \
TMT_PLUGIN_REPORT_POLARION_TEMPLATE \
TMT_PLUGIN_REPORT_POLARION_PLANNED_IN \
TMT_PLUGIN_REPORT_POLARION_ASSIGNEE \
TMT_PLUGIN_REPORT_POLARION_POOL_TEAM \
TMT_PLUGIN_REPORT_POLARION_ARCH \
TMT_PLUGIN_REPORT_POLARION_PLATFORM \
TMT_PLUGIN_REPORT_POLARION_BUILD \
TMT_PLUGIN_REPORT_POLARION_SAMPLE_IMAGE \
TMT_PLUGIN_REPORT_POLARION_LOGS \
TMT_PLUGIN_REPORT_POLARION_COMPOSE_ID \
TMT_PLUGIN_REPORT_POLARION_USE_FACTS \
"

# The facts must not be set
rlRun "export TMT_PLUGIN_REPORT_POLARION_USE_FACTS=0"
rlRun "tmt run -avr execute report -h polarion --no-upload --project-id RHELBASEOS --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlAssertNotGrep '<property name="polarion-custom-arch" value="' "xunit.xml"
rlAssertNotGrep '<property name="polarion-custom-hostname" value="' "xunit.xml"

# The facts must be set
rlRun "export TMT_PLUGIN_REPORT_POLARION_USE_FACTS=1"
rlRun "tmt run -avr execute report -h polarion --no-upload --project-id RHELBASEOS --file xunit.xml 2>&1 >/dev/null | tee output" 2
rlAssertGrep '<property name="polarion-custom-arch" value="' "xunit.xml"
rlAssertGrep '<property name="polarion-custom-hostname" value="' "xunit.xml"
rlPhaseEnd

rlPhaseStartCleanup
Expand Down
31 changes: 22 additions & 9 deletions tmt/steps/report/junit.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,11 @@ def duration_to_seconds(duration: Optional[str]) -> Optional[int]:
raise tmt.utils.ReportError(f"Malformed duration '{duration}'.") from error


class ResultWrapper:
class ImplementProperties:
"""
The context wrapper for :py:class:`tmt.Result`.
Define a properties attribute.
Adds possibility to wrap the :py:class:`tmt.Result` and dynamically add more attributes which
get available inside the template context.
This class can be used to easily add properties attribute by inheriting it.
"""

class PropertyDict(TypedDict):
Expand All @@ -83,24 +82,36 @@ class PropertyDict(TypedDict):
name: str
value: str

def __init__(self, wrapped: tmt.Result) -> None:
self._wrapped = wrapped
def __init__(self) -> None:
self._properties: dict[str, str] = {}

@property
def properties(self) -> list[PropertyDict]:
return [{'name': k, 'value': v} for k, v in self._properties.items()]

@properties.setter
def properties(self, keyval: dict[str, str]) -> None:
self._properties = keyval
def properties(self, keyval: dict[str, Optional[str]]) -> None:
self._properties = {k: v for k, v in keyval.items() if v is not None}


class ResultWrapper(ImplementProperties):
"""
The context wrapper for :py:class:`tmt.Result`.
Adds possibility to wrap the :py:class:`tmt.Result` and dynamically add more attributes which
get available inside the template context.
"""

def __init__(self, wrapped: tmt.Result) -> None:
super().__init__()
self._wrapped = wrapped

def __getattr__(self, name: str) -> Any:
""" Returns an attribute of a wrapped ``tmt.Result`` instance """
return getattr(self._wrapped, name)


class ResultsContext:
class ResultsContext(ImplementProperties):
"""
The results context for Jinja templates.
Expand All @@ -109,6 +120,8 @@ class ResultsContext:
"""

def __init__(self, results: list[tmt.Result]) -> None:
super().__init__()

# Decorate all the tmt.Results with more attributes
self._results: list[ResultWrapper] = [ResultWrapper(r) for r in results]

Expand Down
97 changes: 97 additions & 0 deletions tmt/steps/report/junit/schemas/polarion.xsd
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
<?xml version="1.0" encoding="UTF-8" ?>

<!--
This schema supports only a subset of the features provided by the
`xml-junit` library. Additionally, many attributes are explicitly set as
required. This is intentional to limit the currently supported features of
the tmt Polarion report plugin .
The Polarion `xunit.xml` is almost the same as default output of junit
report plugin but it must allow definition of `properties` inside of
`testsuites` (NOT `testsuite`) and `testcase`.
-->

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">

<xs:element name="failure">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="error">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="skipped">
<xs:complexType mixed="true">
<xs:attribute name="type" type="xs:string" use="required"/>
<xs:attribute name="message" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
<xs:element name="system-err" type="xs:string"/>
<xs:element name="system-out" type="xs:string"/>

<xs:element name="properties">
<xs:complexType>
<xs:sequence>
<xs:element ref="property" maxOccurs="unbounded"/>
</xs:sequence>
</xs:complexType>
</xs:element>

<xs:element name="property">
<xs:complexType>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="value" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="testcase">
<xs:complexType>
<xs:sequence>
<xs:element ref="skipped" minOccurs="0" maxOccurs="1"/>
<xs:element ref="error" minOccurs="0" maxOccurs="1"/>
<xs:element ref="failure" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-out" minOccurs="0" maxOccurs="1"/>
<xs:element ref="system-err" minOccurs="0" maxOccurs="1"/>
<xs:element ref="properties" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:float" use="optional"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuite">
<xs:complexType>
<xs:sequence>
<xs:element ref="testcase" minOccurs="0" maxOccurs="unbounded"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="tests" type="xs:string" use="required"/>
<xs:attribute name="failures" type="xs:string" use="required"/>
<xs:attribute name="errors" type="xs:string" use="required"/>
<xs:attribute name="disabled" type="xs:string" use="required"/>
<xs:attribute name="skipped" type="xs:string" use="required"/>
<xs:attribute name="time" type="xs:float" use="required"/>
</xs:complexType>
</xs:element>

<xs:element name="testsuites">
<xs:complexType>
<xs:sequence>
<xs:element ref="testsuite" minOccurs="1" maxOccurs="1"/>
<xs:element ref="properties" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
<xs:attribute name="time" type="xs:float" use="required"/>
<xs:attribute name="tests" type="xs:string" use="optional"/>
<xs:attribute name="failures" type="xs:string" use="optional"/>
<xs:attribute name="disabled" type="xs:string" use="optional"/>
<xs:attribute name="errors" type="xs:string" use="optional"/>
</xs:complexType>
</xs:element>
</xs:schema>
14 changes: 14 additions & 0 deletions tmt/steps/report/junit/templates/_base.xml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,24 @@
{% if INCLUDE_OUTPUT_LOG and main_log %}
<system-out>{{ main_log | e }}</system-out>
{% endif %}

{# Optionally add the result properties #}
{% if result.properties is defined %}
{% with properties=result.properties %}
{% include "includes/_properties.xml.j2" %}
{% endwith %}
{% endif %}
</testcase>
{% endfor %}
{% endblock %}
</testsuite>
{% endblock %}

{# Optionally include the properties section in testsuites tag #}
{% if RESULTS.properties is defined %}
{% with properties=RESULTS.properties %}
{% include "includes/_properties.xml.j2" %}
{% endwith %}
{% endif %}
</testsuites>
{% endblock %}
7 changes: 7 additions & 0 deletions tmt/steps/report/junit/templates/includes/_properties.xml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{% if properties %}
<properties>
{% for property in properties %}
<property name="{{ property.name | e }}" value="{{ property.value | e }}"/>
{% endfor %}
</properties>
{% endif %}
1 change: 1 addition & 0 deletions tmt/steps/report/junit/templates/polarion.xml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{% extends "_base.xml.j2" %}
Loading

0 comments on commit 3075f79

Please sign in to comment.