Add tests for custom template with subresults

tests/report/junit/data/beaker-phases-subresults.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+. /usr/share/beakerlib/beakerlib.sh || exit 1
+
+
+rlJournalStart
+ rlPhaseStartSetup "phase-setup"
+ rlRun "tmp=\$(mktemp -d)" 0 "Create tmp directory"
+ rlRun "pushd $tmp"
+ rlPhaseEnd
+
+ rlPhaseStartTest "phase-test pass"
+ rlRun -s "echo mytest-pass" 0 "Check output"
+ rlAssertGrep "mytest-pass" "$rlRun_LOG"
+ rlPhaseEnd
+
+ rlPhaseStartTest "phase-test fail"
+ rlRun -s "echo mytest-fail" 0 "Check output"
+ rlAssertGrep "asdf-asdf" "$rlRun_LOG"
+ rlPhaseEnd
+
+ rlPhaseStartTest "phase-test multiple tmt-report-result"
+ # This will create more subresults for each tmt-report-result call
+ rlRun "tmt-report-result extra-tmt-report-result/good PASS"
+ rlRun "tmt-report-result extra-tmt-report-result/bad FAIL"
+ rlRun "tmt-report-result extra-tmt-report-result/weird WARN"
+ rlRun "tmt-report-result extra-tmt-report-result/skip SKIP"
+ rlPhaseEnd
+
+ rlPhaseStartCleanup "phase-cleanup"
+ rlRun "popd"
+ rlRun "rm -r $tmp" 0 "Remove tmp directory"
+ rlPhaseEnd
+rlJournalEnd

tests/report/junit/data/custom-subresults.xml.j2

@@ -0,0 +1,14 @@
+<?xml version="1.0" ?>
+
+<tests disabled="0" errors="{{ RESULTS.errored | length }}" failures="{{ RESULTS.failed | length }}" tests="{{ RESULTS | length }}" time="{{ RESULTS.duration }}">
+ {% for result in RESULTS %}
+ {% set main_test_duration = result.duration | duration_to_seconds %}
+ <result name="{{ result.name | trim | e }}" disabled="0" errors="{{ result.subresult.errored | length }}" failures="{{ result.subresult.failed | length }}" skipped="{{ result.subresult.skipped | length }}" tests="{{ result.subresult | length }}" time="{{ main_test_duration }}" outcome="{{ result.result.value }}" timestamp="{{ result.start_time }}">
+ {% for subresult in result.subresult %}
+ {% set subresult_test_duration = subresult.duration | duration_to_seconds %}
+ <subresult name="{{ subresult.name | e }}" {% if subresult_test_duration %}time="{{ subresult_test_duration }}"{% endif %} outcome="{{ subresult.result.value }}"></subresult>
+ {% endfor %}
+ </result>
+
+ {% endfor %}
+</tests>

tests/report/junit/data/main.fmf

@@ -2,7 +2,7 @@
  discover:
  how: fmf
  provision:
- how: local
+ how: container
  execute:
  how: tmt
 
@@ -17,6 +17,8 @@
  /timeout:
  test: ./runtest.sh timeout
  duration: 2s
+ /subresults:
+ test: ./beaker-phases-subresults.sh
 
  /shell:
  framework: shell
@@ -29,3 +31,30 @@
  duration: 2s
  /escape"<speci&l>_chars:
  test: ./special_chars.sh
+
+ /subresults/pass:
+ summary: Basic pass test of shell subresults
+ test: |
+ tmt-report-result /pass-subtest/good0 PASS
+ tmt-report-result /pass-subtest/good1 PASS
+ tmt-report-result /pass-subtest/good2 PASS
+
+ /subresults/skip:
+ summary: Test with skip and pass test outcomes should pass
+ test: |
+ tmt-report-result /skip-subtest/extra-skip SKIP
+ tmt-report-result /skip-subtest/extra-pass PASS
+
+ /subresults/fail:
+ summary: This reduced outcome of shell subresults must be fail
+ test: |
+ tmt-report-result /fail-subtest/good PASS
+ tmt-report-result /fail-subtest/fail FAIL
+ tmt-report-result /fail-subtest/weird WARN
+ tmt-report-result /fail-subtest/skip SKIP
+
+ /subresults/sleep:
+ summary: Add some sleep to test the duration
+ test: |
+ sleep 2; tmt-report-result /sleep-subtest/good PASS
+ sleep 3; tmt-report-result /sleep-subtest/bad FAIL

tests/report/junit/test.sh

@@ -10,9 +10,9 @@ rlJournalStart
  for method in tmt; do
  rlPhaseStartTest "[$method] Basic format checks"
  rlRun "tmt run -avr execute -h $method report -h junit --file junit.xml 2>&1 >/dev/null | tee output" 2
- rlAssertGrep "3 tests passed, 2 tests failed and 2 errors" "output"
+ rlAssertGrep "5 tests passed, 5 tests failed and 2 errors" "output"
  rlAssertGrep '00:00:00 pass /test/shell/escape"<speci&l>_chars (on default-0)' "output"
- rlAssertGrep '<testsuite name="/plan" disabled="0" errors="2" failures="2" skipped="0" tests="7"' "junit.xml"
+ rlAssertGrep '<testsuite name="/plan" disabled="0" errors="2" failures="5" skipped="0" tests="12"' "junit.xml"
  rlAssertGrep 'fail</failure>' "junit.xml"
 
  # Test the escape of special characters
@@ -25,7 +25,7 @@ rlJournalStart
 
  rlPhaseStartTest "[$method] Check the flavor argument is working"
  rlRun "tmt run -avr execute -h $method report -h junit --file junit.xml --flavor default 2>&1 >/dev/null | tee output" 2
- rlAssertGrep "3 tests passed, 2 tests failed and 2 errors" "output"
+ rlAssertGrep "5 tests passed, 5 tests failed and 2 errors" "output"
 
  # Check there is no schema problem reported
  rlAssertNotGrep 'The generated XML output is not a valid XML file or it is not valid against the XSD schema\.' "output"
@@ -58,10 +58,34 @@ rlJournalStart
 
  rlAssertGrep 'The generated XML output is not a valid XML file.' "output"
  rlPhaseEnd
+
+ rlPhaseStartTest "[$method] Check the 'custom' flavor and context for subresults"
+ rlRun "tmt run -avr execute -h $method report -h junit --file custom-subresults-template-out.xml --template-path custom-subresults.xml.j2 --flavor custom 2>&1 >/dev/null | tee output" 2
+
+ # Beakerlib subresults
+ rlAssertGrep '<subresult name="/Test" outcome="fail"/>' "custom-subresults-template-out.xml"
+ rlAssertGrep '<subresult name="/Test" outcome="pass"/>' "custom-subresults-template-out.xml"
+ rlAssertGrep '<subresult name="/phase-setup" outcome="pass"/>' "custom-subresults-template-out.xml"
+ rlAssertGrep '<subresult name="/phase-test-pass" outcome="pass"/>' "custom-subresults-template-out.xml"
+ rlAssertGrep '<subresult name="/phase-test-fail" outcome="fail"/>' "custom-subresults-template-out.xml"
+ rlAssertGrep '<subresult name="/extra-tmt-report-result/good" outcome="pass"/>' "custom-subresults-template-out.xml"
+ rlAssertGrep '<subresult name="/extra-tmt-report-result/bad" outcome="fail"/>' "custom-subresults-template-out.xml"
+ rlAssertGrep '<subresult name="/extra-tmt-report-result/weird" outcome="warn"/>' "custom-subresults-template-out.xml"
+ rlAssertGrep '<subresult name="/extra-tmt-report-result/skip" outcome="skip"/>' "custom-subresults-template-out.xml"
+
+ # Chosen shell subresults
+ rlAssertGrep '<subresult name="/fail-subtest/good" outcome="pass"/>' "custom-subresults-template-out.xml"
+ rlAssertGrep '<subresult name="/pass-subtest/good0" outcome="pass"/>' "custom-subresults-template-out.xml"
+ rlAssertGrep '<subresult name="/skip-subtest/extra-skip" outcome="skip"/>' "custom-subresults-template-out.xml"
+ rlAssertGrep '<result name="/test/shell/subresults/skip" disabled="0" errors="0" failures="0" skipped="1" tests="2" time="0" outcome="pass"' "custom-subresults-template-out.xml"
+ rlAssertGrep '<result name="/test/shell/subresults/sleep" disabled="0" errors="0" failures="1" skipped="0" tests="2" time="5"' "custom-subresults-template-out.xml"
+
+ rlAssertGrep '<subresult name="/fail-subtest/good" outcome="pass"/>' "custom-subresults-template-out.xml"
+ rlPhaseEnd
  done
 
  rlPhaseStartCleanup
- rlRun "rm output junit.xml custom-template-out.xml"
+ rlRun "rm output junit.xml custom-template-out.xml custom-subresults-template-out.xml"
  rlRun "popd"
  rlPhaseEnd
 rlJournalEnd