From 15b2aa507a084f55dc21bf326d8800c9cf6ce4ad Mon Sep 17 00:00:00 2001 From: Jonas Amundsen Date: Sun, 23 Jun 2024 19:52:15 +0200 Subject: [PATCH] Include skipped (not omitted) tests in reports This fixes #1041 [1]. [1] https://github.com/badeball/cypress-cucumber-preprocessor/issues/1041 --- CHANGELOG.md | 4 + features/fixtures/skipped-all-scenarios.json | 100 ++++++++++++ .../fixtures/skipped-all-scenarios.ndjson | 24 +++ features/fixtures/skipped-first-scenario.json | 90 +++++++++++ .../fixtures/skipped-first-scenario.ndjson | 24 +++ .../fixtures/skipped-second-scenario.json | 90 +++++++++++ .../fixtures/skipped-second-scenario.ndjson | 24 +++ features/fixtures/skipped-third-scenario.json | 90 +++++++++++ .../fixtures/skipped-third-scenario.ndjson | 24 +++ features/issues/1041.feature | 27 ++++ features/pretty_output.feature | 147 ++++++++++++++++++ features/reporters/json.feature | 87 +++++++++++ features/reporters/messages.feature | 87 +++++++++++ features/step_definitions/html_steps.ts | 23 +++ lib/browser-runtime.ts | 88 +++++++++++ 15 files changed, 929 insertions(+) create mode 100644 features/fixtures/skipped-all-scenarios.json create mode 100644 features/fixtures/skipped-all-scenarios.ndjson create mode 100644 features/fixtures/skipped-first-scenario.json create mode 100644 features/fixtures/skipped-first-scenario.ndjson create mode 100644 features/fixtures/skipped-second-scenario.json create mode 100644 features/fixtures/skipped-second-scenario.ndjson create mode 100644 features/fixtures/skipped-third-scenario.json create mode 100644 features/fixtures/skipped-third-scenario.ndjson create mode 100644 features/issues/1041.feature diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ba53651..c9ceaf1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. +## Unreleased + +- Include skipped (not omitted) tests in reports, fixes [#1041](https://github.com/badeball/cypress-cucumber-preprocessor/issues/1041). + ## v20.0.7 - Updated all dependencies, fixes [#1198](https://github.com/badeball/cypress-cucumber-preprocessor/issues/1198). diff --git a/features/fixtures/skipped-all-scenarios.json b/features/fixtures/skipped-all-scenarios.json new file mode 100644 index 00000000..bc4c86d1 --- /dev/null +++ b/features/fixtures/skipped-all-scenarios.json @@ -0,0 +1,100 @@ +[ + { + "description": "", + "elements": [ + { + "description": "", + "id": "a-feature;first-scenario", + "keyword": "Scenario", + "line": 3, + "name": "first scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 4, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "skipped", + "duration": 0 + } + } + ], + "tags": [ + { + "name": "@skip", + "line": 2 + } + ], + "type": "scenario" + }, + { + "description": "", + "id": "a-feature;second-scenario", + "keyword": "Scenario", + "line": 6, + "name": "second scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 7, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "skipped", + "duration": 0 + } + } + ], + "tags": [ + { + "name": "@skip", + "line": 5 + } + ], + "type": "scenario" + }, + { + "description": "", + "id": "a-feature;third-scenario", + "keyword": "Scenario", + "line": 9, + "name": "third scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 10, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "skipped", + "duration": 0 + } + } + ], + "tags": [ + { + "name": "@skip", + "line": 8 + } + ], + "type": "scenario" + } + ], + "id": "a-feature", + "line": 1, + "keyword": "Feature", + "name": "a feature", + "tags": [], + "uri": "cypress/e2e/a.feature" + } +] diff --git a/features/fixtures/skipped-all-scenarios.ndjson b/features/fixtures/skipped-all-scenarios.ndjson new file mode 100644 index 00000000..7c629fda --- /dev/null +++ b/features/fixtures/skipped-all-scenarios.ndjson @@ -0,0 +1,24 @@ +{"meta":"meta"} +{"testRunStarted":{"timestamp":{"seconds":0,"nanos":0}}} +{"source":{"data":"Feature: a feature\n @skip\n Scenario: first scenario\n Given a step\n @skip\n Scenario: second scenario\n Given a step\n @skip\n Scenario: third scenario\n Given a step","uri":"cypress/e2e/a.feature","mediaType":"text/x.cucumber.gherkin+plain"}} +{"gherkinDocument":{"feature":{"tags":[],"location":{"line":1,"column":1},"language":"en","keyword":"Feature","name":"a feature","description":"","children":[{"scenario":{"id":"id","tags":[{"location":{"line":2,"column":3},"name":"@skip","id":"id"}],"location":{"line":3,"column":3},"keyword":"Scenario","name":"first scenario","description":"","steps":[{"id":"id","location":{"line":4,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}},{"scenario":{"id":"id","tags":[{"location":{"line":5,"column":3},"name":"@skip","id":"id"}],"location":{"line":6,"column":3},"keyword":"Scenario","name":"second scenario","description":"","steps":[{"id":"id","location":{"line":7,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}},{"scenario":{"id":"id","tags":[{"location":{"line":8,"column":3},"name":"@skip","id":"id"}],"location":{"line":9,"column":3},"keyword":"Scenario","name":"third scenario","description":"","steps":[{"id":"id","location":{"line":10,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}}]},"comments":[],"uri":"cypress/e2e/a.feature"}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[{"name":"@skip","astNodeId":"id"}],"name":"first scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[{"name":"@skip","astNodeId":"id"}],"name":"second scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[{"name":"@skip","astNodeId":"id"}],"name":"third scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"stepDefinition":{"id":"id","pattern":{"type":"CUCUMBER_EXPRESSION","source":"a step"},"sourceReference":{"uri":"not available","location":{"line":0}}}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"SKIPPED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"SKIPPED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"SKIPPED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testRunFinished":{"timestamp":{"seconds":0,"nanos":0}}} diff --git a/features/fixtures/skipped-first-scenario.json b/features/fixtures/skipped-first-scenario.json new file mode 100644 index 00000000..cd4184fa --- /dev/null +++ b/features/fixtures/skipped-first-scenario.json @@ -0,0 +1,90 @@ +[ + { + "description": "", + "elements": [ + { + "description": "", + "id": "a-feature;first-scenario", + "keyword": "Scenario", + "line": 3, + "name": "first scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 4, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "skipped", + "duration": 0 + } + } + ], + "tags": [ + { + "name": "@skip", + "line": 2 + } + ], + "type": "scenario" + }, + { + "description": "", + "id": "a-feature;second-scenario", + "keyword": "Scenario", + "line": 5, + "name": "second scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 6, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "passed", + "duration": 0 + } + } + ], + "tags": [], + "type": "scenario" + }, + { + "description": "", + "id": "a-feature;third-scenario", + "keyword": "Scenario", + "line": 7, + "name": "third scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 8, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "passed", + "duration": 0 + } + } + ], + "tags": [], + "type": "scenario" + } + ], + "id": "a-feature", + "line": 1, + "keyword": "Feature", + "name": "a feature", + "tags": [], + "uri": "cypress/e2e/a.feature" + } +] diff --git a/features/fixtures/skipped-first-scenario.ndjson b/features/fixtures/skipped-first-scenario.ndjson new file mode 100644 index 00000000..6af0d968 --- /dev/null +++ b/features/fixtures/skipped-first-scenario.ndjson @@ -0,0 +1,24 @@ +{"meta":"meta"} +{"testRunStarted":{"timestamp":{"seconds":0,"nanos":0}}} +{"source":{"data":"Feature: a feature\n @skip\n Scenario: first scenario\n Given a step\n Scenario: second scenario\n Given a step\n Scenario: third scenario\n Given a step","uri":"cypress/e2e/a.feature","mediaType":"text/x.cucumber.gherkin+plain"}} +{"gherkinDocument":{"feature":{"tags":[],"location":{"line":1,"column":1},"language":"en","keyword":"Feature","name":"a feature","description":"","children":[{"scenario":{"id":"id","tags":[{"location":{"line":2,"column":3},"name":"@skip","id":"id"}],"location":{"line":3,"column":3},"keyword":"Scenario","name":"first scenario","description":"","steps":[{"id":"id","location":{"line":4,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}},{"scenario":{"id":"id","tags":[],"location":{"line":5,"column":3},"keyword":"Scenario","name":"second scenario","description":"","steps":[{"id":"id","location":{"line":6,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}},{"scenario":{"id":"id","tags":[],"location":{"line":7,"column":3},"keyword":"Scenario","name":"third scenario","description":"","steps":[{"id":"id","location":{"line":8,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}}]},"comments":[],"uri":"cypress/e2e/a.feature"}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[{"name":"@skip","astNodeId":"id"}],"name":"first scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[],"name":"second scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[],"name":"third scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"stepDefinition":{"id":"id","pattern":{"type":"CUCUMBER_EXPRESSION","source":"a step"},"sourceReference":{"uri":"not available","location":{"line":0}}}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"SKIPPED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"PASSED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"PASSED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testRunFinished":{"timestamp":{"seconds":0,"nanos":0}}} diff --git a/features/fixtures/skipped-second-scenario.json b/features/fixtures/skipped-second-scenario.json new file mode 100644 index 00000000..b776ca89 --- /dev/null +++ b/features/fixtures/skipped-second-scenario.json @@ -0,0 +1,90 @@ +[ + { + "description": "", + "elements": [ + { + "description": "", + "id": "a-feature;first-scenario", + "keyword": "Scenario", + "line": 2, + "name": "first scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 3, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "passed", + "duration": 0 + } + } + ], + "tags": [], + "type": "scenario" + }, + { + "description": "", + "id": "a-feature;second-scenario", + "keyword": "Scenario", + "line": 5, + "name": "second scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 6, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "skipped", + "duration": 0 + } + } + ], + "tags": [ + { + "name": "@skip", + "line": 4 + } + ], + "type": "scenario" + }, + { + "description": "", + "id": "a-feature;third-scenario", + "keyword": "Scenario", + "line": 7, + "name": "third scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 8, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "passed", + "duration": 0 + } + } + ], + "tags": [], + "type": "scenario" + } + ], + "id": "a-feature", + "line": 1, + "keyword": "Feature", + "name": "a feature", + "tags": [], + "uri": "cypress/e2e/a.feature" + } +] diff --git a/features/fixtures/skipped-second-scenario.ndjson b/features/fixtures/skipped-second-scenario.ndjson new file mode 100644 index 00000000..5f9fdc01 --- /dev/null +++ b/features/fixtures/skipped-second-scenario.ndjson @@ -0,0 +1,24 @@ +{"meta":"meta"} +{"testRunStarted":{"timestamp":{"seconds":0,"nanos":0}}} +{"source":{"data":"Feature: a feature\n Scenario: first scenario\n Given a step\n @skip\n Scenario: second scenario\n Given a step\n Scenario: third scenario\n Given a step","uri":"cypress/e2e/a.feature","mediaType":"text/x.cucumber.gherkin+plain"}} +{"gherkinDocument":{"feature":{"tags":[],"location":{"line":1,"column":1},"language":"en","keyword":"Feature","name":"a feature","description":"","children":[{"scenario":{"id":"id","tags":[],"location":{"line":2,"column":3},"keyword":"Scenario","name":"first scenario","description":"","steps":[{"id":"id","location":{"line":3,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}},{"scenario":{"id":"id","tags":[{"location":{"line":4,"column":3},"name":"@skip","id":"id"}],"location":{"line":5,"column":3},"keyword":"Scenario","name":"second scenario","description":"","steps":[{"id":"id","location":{"line":6,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}},{"scenario":{"id":"id","tags":[],"location":{"line":7,"column":3},"keyword":"Scenario","name":"third scenario","description":"","steps":[{"id":"id","location":{"line":8,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}}]},"comments":[],"uri":"cypress/e2e/a.feature"}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[],"name":"first scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[{"name":"@skip","astNodeId":"id"}],"name":"second scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[],"name":"third scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"stepDefinition":{"id":"id","pattern":{"type":"CUCUMBER_EXPRESSION","source":"a step"},"sourceReference":{"uri":"not available","location":{"line":0}}}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"PASSED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"SKIPPED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"PASSED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testRunFinished":{"timestamp":{"seconds":0,"nanos":0}}} diff --git a/features/fixtures/skipped-third-scenario.json b/features/fixtures/skipped-third-scenario.json new file mode 100644 index 00000000..6af48878 --- /dev/null +++ b/features/fixtures/skipped-third-scenario.json @@ -0,0 +1,90 @@ +[ + { + "description": "", + "elements": [ + { + "description": "", + "id": "a-feature;first-scenario", + "keyword": "Scenario", + "line": 2, + "name": "first scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 3, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "passed", + "duration": 0 + } + } + ], + "tags": [], + "type": "scenario" + }, + { + "description": "", + "id": "a-feature;second-scenario", + "keyword": "Scenario", + "line": 4, + "name": "second scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 5, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "passed", + "duration": 0 + } + } + ], + "tags": [], + "type": "scenario" + }, + { + "description": "", + "id": "a-feature;third-scenario", + "keyword": "Scenario", + "line": 7, + "name": "third scenario", + "steps": [ + { + "arguments": [], + "keyword": "Given ", + "line": 8, + "name": "a step", + "match": { + "location": "not available:0" + }, + "result": { + "status": "skipped", + "duration": 0 + } + } + ], + "tags": [ + { + "name": "@skip", + "line": 6 + } + ], + "type": "scenario" + } + ], + "id": "a-feature", + "line": 1, + "keyword": "Feature", + "name": "a feature", + "tags": [], + "uri": "cypress/e2e/a.feature" + } +] diff --git a/features/fixtures/skipped-third-scenario.ndjson b/features/fixtures/skipped-third-scenario.ndjson new file mode 100644 index 00000000..d29bf06e --- /dev/null +++ b/features/fixtures/skipped-third-scenario.ndjson @@ -0,0 +1,24 @@ +{"meta":"meta"} +{"testRunStarted":{"timestamp":{"seconds":0,"nanos":0}}} +{"source":{"data":"Feature: a feature\n Scenario: first scenario\n Given a step\n Scenario: second scenario\n Given a step\n @skip\n Scenario: third scenario\n Given a step","uri":"cypress/e2e/a.feature","mediaType":"text/x.cucumber.gherkin+plain"}} +{"gherkinDocument":{"feature":{"tags":[],"location":{"line":1,"column":1},"language":"en","keyword":"Feature","name":"a feature","description":"","children":[{"scenario":{"id":"id","tags":[],"location":{"line":2,"column":3},"keyword":"Scenario","name":"first scenario","description":"","steps":[{"id":"id","location":{"line":3,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}},{"scenario":{"id":"id","tags":[],"location":{"line":4,"column":3},"keyword":"Scenario","name":"second scenario","description":"","steps":[{"id":"id","location":{"line":5,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}},{"scenario":{"id":"id","tags":[{"location":{"line":6,"column":3},"name":"@skip","id":"id"}],"location":{"line":7,"column":3},"keyword":"Scenario","name":"third scenario","description":"","steps":[{"id":"id","location":{"line":8,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}}]},"comments":[],"uri":"cypress/e2e/a.feature"}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[],"name":"first scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[],"name":"second scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"pickle":{"id":"id","uri":"cypress/e2e/a.feature","astNodeIds":["id"],"tags":[{"name":"@skip","astNodeId":"id"}],"name":"third scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}} +{"stepDefinition":{"id":"id","pattern":{"type":"CUCUMBER_EXPRESSION","source":"a step"},"sourceReference":{"uri":"not available","location":{"line":0}}}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"PASSED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"PASSED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}} +{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}} +{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"SKIPPED","duration":0},"timestamp":{"seconds":0,"nanos":0}}} +{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}} +{"testRunFinished":{"timestamp":{"seconds":0,"nanos":0}}} diff --git a/features/issues/1041.feature b/features/issues/1041.feature new file mode 100644 index 00000000..27c354cd --- /dev/null +++ b/features/issues/1041.feature @@ -0,0 +1,27 @@ +# https://github.com/badeball/cypress-cucumber-preprocessor/issues/1034 + +Feature: HTML report + Scenario: skipepd test + Given additional preprocessor configuration + """ + { + "html": { + "enabled": true + } + } + """ + And a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + @skip + Scenario: a scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function() {}); + """ + When I run cypress + Then it passes + And the HTML report should display 1 "skipped" scenario diff --git a/features/pretty_output.feature b/features/pretty_output.feature index cf96e9d7..2e102b66 100644 --- a/features/pretty_output.feature +++ b/features/pretty_output.feature @@ -438,6 +438,153 @@ Feature: pretty output Given a step """ + Rule: it should include results of skipped (not omitted) tests + Scenario: first scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + @skip + Scenario: first scenario + Given a step + Scenario: second scenario + Given a step + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And the output should contain + """ + Feature: a feature # cypress/e2e/a.feature:1 + + @skip + Scenario: first scenario # cypress/e2e/a.feature:3 + Given a step + - skipped + + Scenario: second scenario # cypress/e2e/a.feature:5 + Given a step + + Scenario: third scenario # cypress/e2e/a.feature:7 + Given a step + """ + + Scenario: middle scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + Scenario: first scenario + Given a step + @skip + Scenario: second scenario + Given a step + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And the output should contain + """ + Feature: a feature # cypress/e2e/a.feature:1 + + Scenario: first scenario # cypress/e2e/a.feature:2 + Given a step + + @skip + Scenario: second scenario # cypress/e2e/a.feature:5 + Given a step + - skipped + + Scenario: third scenario # cypress/e2e/a.feature:7 + Given a step + """ + + Scenario: last scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + Scenario: first scenario + Given a step + Scenario: second scenario + Given a step + @skip + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And the output should contain + """ + Feature: a feature # cypress/e2e/a.feature:1 + + Scenario: first scenario # cypress/e2e/a.feature:2 + Given a step + + Scenario: second scenario # cypress/e2e/a.feature:4 + Given a step + + @skip + Scenario: third scenario # cypress/e2e/a.feature:7 + Given a step + - skipped + """ + + Scenario: all scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + @skip + Scenario: first scenario + Given a step + @skip + Scenario: second scenario + Given a step + @skip + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And the output should contain + """ + Feature: a feature # cypress/e2e/a.feature:1 + + @skip + Scenario: first scenario # cypress/e2e/a.feature:3 + Given a step + - skipped + + @skip + Scenario: second scenario # cypress/e2e/a.feature:6 + Given a step + - skipped + + @skip + Scenario: third scenario # cypress/e2e/a.feature:9 + Given a step + - skipped + """ + @network Rule: it should handle reloads gracefully in a multitude of scenarios diff --git a/features/reporters/json.feature b/features/reporters/json.feature index a0bd9914..c99ddc28 100644 --- a/features/reporters/json.feature +++ b/features/reporters/json.feature @@ -387,6 +387,93 @@ Feature: JSON formatter Then it fails And the JSON report should contain an image attachment for what appears to be a screenshot + Rule: it should include results of skipped (not omitted) tests + Scenario: first scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + @skip + Scenario: first scenario + Given a step + Scenario: second scenario + Given a step + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And there should be a JSON output similar to "fixtures/skipped-first-scenario.json" + + Scenario: middle scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + Scenario: first scenario + Given a step + @skip + Scenario: second scenario + Given a step + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And there should be a JSON output similar to "fixtures/skipped-second-scenario.json" + + Scenario: last scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + Scenario: first scenario + Given a step + Scenario: second scenario + Given a step + @skip + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And there should be a JSON output similar to "fixtures/skipped-third-scenario.json" + + Scenario: all scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + @skip + Scenario: first scenario + Given a step + @skip + Scenario: second scenario + Given a step + @skip + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And there should be a JSON output similar to "fixtures/skipped-all-scenarios.json" + Rule: failing Cypress hooks outside of the plugins control should discontinue the report Scenario: failing before hook Given a file named "cypress/e2e/a.feature" with: diff --git a/features/reporters/messages.feature b/features/reporters/messages.feature index e583f899..c0c76388 100644 --- a/features/reporters/messages.feature +++ b/features/reporters/messages.feature @@ -388,6 +388,93 @@ Feature: messages report Then it fails And the messages report should contain an image attachment for what appears to be a screenshot + Rule: it should include results of skipped (not omitted) tests + Scenario: first scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + @skip + Scenario: first scenario + Given a step + Scenario: second scenario + Given a step + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And there should be a messages similar to "fixtures/skipped-first-scenario.ndjson" + + Scenario: middle scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + Scenario: first scenario + Given a step + @skip + Scenario: second scenario + Given a step + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And there should be a messages similar to "fixtures/skipped-second-scenario.ndjson" + + Scenario: last scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + Scenario: first scenario + Given a step + Scenario: second scenario + Given a step + @skip + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And there should be a messages similar to "fixtures/skipped-third-scenario.ndjson" + + Scenario: all scenario skipped + Given a file named "cypress/e2e/a.feature" with: + """ + Feature: a feature + @skip + Scenario: first scenario + Given a step + @skip + Scenario: second scenario + Given a step + @skip + Scenario: third scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function () {}); + """ + When I run cypress + Then it passes + And there should be a messages similar to "fixtures/skipped-all-scenarios.ndjson" + Rule: failing Cypress hooks outside of the plugins control should discontinue the report Scenario: failing before hook Given a file named "cypress/e2e/a.feature" with: diff --git a/features/step_definitions/html_steps.ts b/features/step_definitions/html_steps.ts index a3bbfd94..03b5e75e 100644 --- a/features/step_definitions/html_steps.ts +++ b/features/step_definitions/html_steps.ts @@ -100,3 +100,26 @@ Then( assert(AccordionItemPanel); } ); + +Then( + "the HTML report should display {int} {string} scenario(s)", + async function (this: ICustomWorld, n: number, status: string) { + const dom = await JSDOM.fromFile( + path.join(this.tmpDir, "cucumber-report.html"), + { runScripts: "dangerously" } + ); + + const li = await findByText( + dom.window.document.documentElement, + new RegExp(`\\d+ ${status}`), + { + selector: "li", + } + ); + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const actual = parseInt(li.textContent!, 10); + + assert.equal(actual, n); + } +); diff --git a/lib/browser-runtime.ts b/lib/browser-runtime.ts index cea30380..d2d737d7 100644 --- a/lib/browser-runtime.ts +++ b/lib/browser-runtime.ts @@ -82,6 +82,7 @@ interface CompositionContext { astIdsMap: ReturnType; testStepIds: TestStepIds; pickles: messages.Pickle[]; + includedPickles: (messages.Pickle & { willBekipped: boolean })[]; specEnvelopes: messages.Envelope[]; testFilter: Node; omitFiltered: boolean; @@ -208,6 +209,66 @@ function taskTestStepFinished( } } +function emitSkippedPickle( + context: CompositionContext, + pickle: messages.Pickle +) { + const { registry } = context; + + const testCaseId = pickle.id; + const pickleSteps = pickle.steps ?? []; + const tags = collectTagNames(pickle.tags); + const beforeHooks = registry.resolveBeforeHooks(tags); + const afterHooks = registry.resolveAfterHooks(tags); + const testCaseStartedId = context.newId(); + const timestamp = createTimestamp(); + + const steps: (ICaseHook | messages.PickleStep)[] = [ + ...beforeHooks, + ...pickleSteps, + ...afterHooks, + ]; + + taskTestCaseStarted(context, { + id: testCaseStartedId, + testCaseId, + attempt: 0, + timestamp, + }); + + for (const step of steps) { + const testStepId = getTestStepId({ + context, + pickleId: pickle.id, + hookIdOrPickleStepId: step.id, + }); + + taskTestStepStarted(context, { + testStepId, + testCaseStartedId, + timestamp, + }); + + taskTestStepFinished(context, { + testStepId, + testCaseStartedId, + testStepResult: { + status: messages.TestStepResultStatus.SKIPPED, + duration: { + seconds: 0, + nanos: 0, + }, + }, + timestamp, + }); + } + taskTestCaseFinished(context, { + testCaseStartedId, + timestamp, + willBeRetried: false, + }); +} + function findPickleById(context: CompositionContext, astId: string) { return assertAndReturn( context.pickles.find( @@ -478,6 +539,14 @@ function createPickle(context: CompositionContext, pickle: messages.Pickle) { } it(scenarioName, inheritedTestOptions, function () { + /** + * This must always be true, otherwise something is off. + */ + assert( + context.includedPickles[0].id === pickle.id, + "Included pickle stack is unsynchronized" + ); + const { remainingSteps, testCaseStartedId } = retrieveInternalSpecProperties(); @@ -796,6 +865,13 @@ function beforeHandler(this: Mocha.Context, context: CompositionContext) { } taskSpecEnvelopes(context); + + while ( + context.includedPickles.length > 0 && + context.includedPickles[0].willBekipped + ) { + emitSkippedPickle(context, context.includedPickles.shift()!); + } } function beforeEachHandler(context: CompositionContext) { @@ -1028,6 +1104,15 @@ function afterEachHandler(this: Mocha.Context, context: CompositionContext) { testCaseStartedId: context.newId(), remainingSteps: [...properties.allSteps], }); + } else { + context.includedPickles.shift(); + + while ( + context.includedPickles.length > 0 && + context.includedPickles[0].willBekipped + ) { + emitSkippedPickle(context, context.includedPickles.shift()!); + } } } @@ -1189,6 +1274,9 @@ export default function createTests( astIdsMap: createAstIdMap(gherkinDocument), testStepIds, pickles, + includedPickles: includedPickles.map((pickle) => { + return { ...pickle, willBekipped: shouldSkipPickle(testFilter, pickle) }; + }), specEnvelopes, testFilter, omitFiltered,