Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore(testrunner): split TestRunner into parts #1679

Merged
merged 1 commit into from
Apr 7, 2020
Merged

chore(testrunner): split TestRunner into parts #1679

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions test/playwright.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ const fs = require('fs');
const path = require('path');
const rm = require('rimraf').sync;
const GoldenUtils = require('./golden-utils');
const {Matchers} = require('../utils/testrunner/');
const {Matchers} = require('../utils/testrunner/Matchers');
const readline = require('readline');
const {TestServer} = require('../utils/testserver/');

Expand Down Expand Up @@ -174,14 +174,13 @@ module.exports.addPlaywrightTests = ({testRunner, platform, products, playwright
state._stderr.close();
});

beforeEach(async(state, test) => {
test.output = [];
const dumpout = data => test.output.push(`\x1b[33m[pw:stdio:out]\x1b[0m ${data}`);
const dumperr = data => test.output.push(`\x1b[31m[pw:stdio:err]\x1b[0m ${data}`);
beforeEach(async(state, testRun) => {
const dumpout = data => testRun.log(`\x1b[33m[pw:stdio:out]\x1b[0m ${data}`);
const dumperr = data => testRun.log(`\x1b[31m[pw:stdio:err]\x1b[0m ${data}`);
state._stdout.on('line', dumpout);
state._stderr.on('line', dumperr);
if (dumpProtocolOnFailure)
state.browser._debugProtocol.log = data => test.output.push(`\x1b[32m[pw:protocol]\x1b[0m ${data}`);
state.browser._debugProtocol.log = data => testRun.log(`\x1b[32m[pw:protocol]\x1b[0m ${data}`);
state.tearDown = async () => {
state._stdout.off('line', dumpout);
state._stderr.off('line', dumperr);
Expand Down
22 changes: 8 additions & 14 deletions test/test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

const {TestRunner, Reporter} = require('../utils/testrunner/');
const TestRunner = require('../utils/testrunner/');
const utils = require('./utils');
const os = require('os');

Expand All @@ -38,9 +38,13 @@ if (MAJOR_NODEJS_VERSION >= 8 && require('inspector').url()) {

const testRunner = new TestRunner({
timeout,
totalTimeout: process.env.CI ? 15 * 60 * 1000 : 0,
parallel,
breakOnFailure: process.argv.indexOf('--break-on-failure') !== -1,
installCommonHelpers: false
verbose: process.argv.includes('--verbose'),
summary: !process.argv.includes('--verbose'),
showSlowTests: process.env.CI ? 5 : 0,
showMarkedAsFailingTests: 10,
});
utils.setupTestRunner(testRunner);

Expand Down Expand Up @@ -68,7 +72,7 @@ require('./playwright.spec.js').addPlaywrightTests({
playwrightPath: utils.projectRoot(),
products,
platform: os.platform(),
testRunner,
testRunner: testRunner.api(),
headless: !!valueFromEnv('HEADLESS', true),
slowMo: valueFromEnv('SLOW_MO', 0),
dumpProtocolOnFailure: valueFromEnv('DEBUGP', false),
Expand All @@ -81,15 +85,5 @@ if (filterArgIndex !== -1) {
testRunner.focusMatchingTests(new RegExp(filter, 'i'));
}

new Reporter(testRunner, {
verbose: process.argv.includes('--verbose'),
summary: !process.argv.includes('--verbose'),
showSlowTests: process.env.CI ? 5 : 0,
showMarkedAsFailingTests: 10,
});

// await utils.initializeFlakinessDashboardIfNeeded(testRunner);
testRunner.run({ totalTimeout: process.env.CI ? 15 * 60 * 1000 : 0 }).then(result => {
process.exit(result.exitCode);
});

testRunner.run();
46 changes: 21 additions & 25 deletions test/utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,8 @@ const utils = module.exports = {
const ignoredMethods = new Set(ignoredMethodsArray);
for (const [className, classType] of Object.entries(api))
traceAPICoverage(coverage, events, className, classType);
const focus = testRunner.hasFocusedTestsOrSuites();
(focus ? testRunner.fdescribe : testRunner.describe)(COVERAGE_TESTSUITE_NAME, () => {
(focus ? testRunner.fit : testRunner.it)('should call all API methods', () => {
testRunner.describe(COVERAGE_TESTSUITE_NAME, () => {
testRunner.it('should call all API methods', () => {
const missingMethods = [];
const extraIgnoredMethods = [];
for (const method of coverage.keys()) {
Expand Down Expand Up @@ -292,49 +291,46 @@ const utils = module.exports = {
},

setupTestRunner: function(testRunner) {
testRunner.testModifier('skip', (t, condition) => condition && t.setSkipped(true));
testRunner.suiteModifier('skip', (s, condition) => condition && s.setSkipped(true));
testRunner.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail));
testRunner.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail));
testRunner.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3));
testRunner.testModifier('repeat', (t, count) => t.setRepeat(count));
testRunner.suiteModifier('repeat', (s, count) => s.setRepeat(count));
testRunner.testAttribute('focus', t => t.setFocused(true));
testRunner.suiteAttribute('focus', s => s.setFocused(true));
testRunner.testAttribute('debug', t => {
t.setFocused(true);
const collector = testRunner._collector;
collector.addTestModifier('skip', (t, condition) => condition && t.setSkipped(true));
collector.addSuiteModifier('skip', (s, condition) => condition && s.setSkipped(true));
collector.addTestModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail));
collector.addSuiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail));
collector.addTestModifier('slow', t => t.setTimeout(t.timeout() * 3));
collector.addTestAttribute('debug', t => {
t.setTimeout(100000000);

let session;
t.before(async () => {
t.environment().beforeEach(async () => {
const inspector = require('inspector');
const readFileAsync = util.promisify(fs.readFile.bind(fs));
session = new require('inspector').Session();
session = new inspector.Session();
session.connect();
const postAsync = util.promisify(session.post.bind(session));
await postAsync('Debugger.enable');
const setBreakpointCommands = [];
const N = t.body().toString().split('\n').length;
const location = t.location();
const lines = (await readFileAsync(location.filePath, 'utf8')).split('\n');
const lines = (await readFileAsync(location.filePath(), 'utf8')).split('\n');
for (let line = 0; line < N; ++line) {
const lineNumber = line + location.lineNumber;
const lineNumber = line + location.lineNumber();
setBreakpointCommands.push(postAsync('Debugger.setBreakpointByUrl', {
url: url.pathToFileURL(location.filePath),
url: url.pathToFileURL(location.filePath()),
lineNumber,
condition: `console.log('${String(lineNumber + 1).padStart(6, ' ')} | ' + ${JSON.stringify(lines[lineNumber])})`,
}).catch(e => {}));
}
await Promise.all(setBreakpointCommands);
});

t.after(async () => {
t.environment().afterEach(async () => {
session.disconnect();
});
});
testRunner.fdescribe = testRunner.describe.focus;
testRunner.xdescribe = testRunner.describe.skip(true);
testRunner.fit = testRunner.it.focus;
testRunner.xit = testRunner.it.skip(true);
testRunner.dit = testRunner.it.debug;
testRunner.api().fdescribe = testRunner.api().describe.only;
testRunner.api().xdescribe = testRunner.api().describe.skip(true);
testRunner.api().fit = testRunner.api().it.only;
testRunner.api().xit = testRunner.api().it.skip(true);
testRunner.api().dit = testRunner.api().it.only.debug;
},
};
24 changes: 12 additions & 12 deletions utils/doclint/check_public_api/test/test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@ const mdBuilder = require('../MDBuilder');
const jsBuilder = require('../JSBuilder');
const GoldenUtils = require('../../../../test/golden-utils');

const {TestRunner, Reporter, Matchers} = require('../../../testrunner/');
const {Matchers} = require('../../../testrunner/Matchers');
const TestRunner = require('../../../testrunner/');
const runner = new TestRunner();
const reporter = new Reporter(runner);

const {describe, xdescribe, fdescribe} = runner;
const {it, fit, xit} = runner;
const {beforeAll, beforeEach, afterAll, afterEach} = runner;
const {describe, xdescribe, fdescribe} = runner.api();
const {it, fit, xit} = runner.api();
const {beforeAll, beforeEach, afterAll, afterEach} = runner.api();

let browserContext;
let browser;
let page;

beforeAll(async function() {
Expand Down Expand Up @@ -60,8 +60,8 @@ describe('checkPublicAPI', function() {

runner.run();

async function testLint(state, test) {
const dirPath = path.join(__dirname, test.name());
async function testLint(state, testRun) {
const dirPath = path.join(__dirname, testRun.test().name());
const {expect} = new Matchers({
toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
});
Expand All @@ -74,8 +74,8 @@ async function testLint(state, test) {
expect(errors.join('\n')).toBeGolden('result.txt');
}

async function testMDBuilder(state, test) {
const dirPath = path.join(__dirname, test.name());
async function testMDBuilder(state, testRun) {
const dirPath = path.join(__dirname, testRun.test().name());
const {expect} = new Matchers({
toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
});
Expand All @@ -84,8 +84,8 @@ async function testMDBuilder(state, test) {
expect(serialize(documentation)).toBeGolden('result.txt');
}

async function testJSBuilder(state, test) {
const dirPath = path.join(__dirname, test.name());
async function testJSBuilder(state, testRun) {
const dirPath = path.join(__dirname, testRun.test().name());
const {expect} = new Matchers({
toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
});
Expand Down
11 changes: 5 additions & 6 deletions utils/doclint/preprocessor/test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,13 @@

const {runCommands, ensureReleasedAPILinks} = require('.');
const Source = require('../Source');
const {TestRunner, Reporter, Matchers} = require('../../testrunner/');
const TestRunner = require('../../testrunner/');
const runner = new TestRunner();
new Reporter(runner);

const {describe, xdescribe, fdescribe} = runner;
const {it, fit, xit} = runner;
const {beforeAll, beforeEach, afterAll, afterEach} = runner;
const {expect} = new Matchers();
const {describe, xdescribe, fdescribe} = runner.api();
const {it, fit, xit} = runner.api();
const {beforeAll, beforeEach, afterAll, afterEach} = runner.api();
const {expect} = runner.api();

describe('ensureReleasedAPILinks', function() {
it('should work with non-release version', function() {
Expand Down
2 changes: 2 additions & 0 deletions utils/testrunner/Location.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ class Location {
return this._fileName + ':' + this._lineNumber + ':' + this._columnNumber;
}

// TODO: static getCallerLocationIn(glob) vs getCallerLocationIgnoring(glob).

static getCallerLocation(filename) {
const error = new Error();
const stackFrames = error.stack.split('\n').slice(1);
Expand Down
28 changes: 14 additions & 14 deletions utils/testrunner/Reporter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,35 +19,33 @@ const colors = require('colors/safe');
const {MatchError} = require('./Matchers.js');

class Reporter {
constructor(runner, options = {}) {
constructor(delegate, options = {}) {
const {
showSlowTests = 3,
showMarkedAsFailingTests = Infinity,
verbose = false,
summary = true,
} = options;
this._filePathToLines = new Map();
this._runner = runner;
this._delegate = delegate;
this._showSlowTests = showSlowTests;
this._showMarkedAsFailingTests = showMarkedAsFailingTests;
this._verbose = verbose;
this._summary = summary;
this._testCounter = 0;
runner.setDelegate(this);
}

onStarted(testRuns) {
this._testCounter = 0;
this._timestamp = Date.now();
const allTests = this._runner.tests();
if (!this._runner.hasFocusedTestsOrSuites()) {
console.log(`Running all ${colors.yellow(testRuns.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
if (!this._delegate.hasFocusedTestsOrSuites()) {
console.log(`Running all ${colors.yellow(testRuns.length)} tests on ${colors.yellow(this._delegate.parallel())} worker${this._delegate.parallel() > 1 ? 's' : ''}:\n`);
} else {
console.log(`Running ${colors.yellow(testRuns.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`);
console.log(`Running ${colors.yellow(testRuns.length)} focused tests out of total ${colors.yellow(this._delegate.testCount())} on ${colors.yellow(this._delegate.parallel())} worker${this._delegate.parallel() > 1 ? 's' : ''}`);
console.log('');
const focusedEntities = [
...this._runner.suites().filter(suite => suite.focused()),
...this._runner.tests().filter(test => test.focused()),
...this._delegate.focusedSuites(),
...this._delegate.focusedTests(),
];
if (focusedEntities.length) {
console.log('Focused Suites and Tests:');
Expand Down Expand Up @@ -174,7 +172,7 @@ class Reporter {
_printVerboseTestRunResult(resultIndex, testRun) {
const test = testRun.test();
let prefix = `${resultIndex})`;
if (this._runner.parallel() > 1)
if (this._delegate.parallel() > 1)
prefix += ' ' + colors.gray(`[worker = ${testRun.workerId()}]`);
if (testRun.result() === 'ok') {
console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName()} (${formatLocation(test.location())})`);
Expand All @@ -187,9 +185,10 @@ class Reporter {
console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName()} (${formatLocation(test.location())})`);
} else if (testRun.result() === 'timedout') {
console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout()}ms]`)} ${test.fullName()} (${formatLocation(test.location())})`);
if (testRun.output) {
const output = testRun.output();
if (output.length) {
console.log(' Output:');
for (const line of testRun.output)
for (const line of output)
console.log(' ' + line);
}
} else if (testRun.result() === 'failed') {
Expand Down Expand Up @@ -235,9 +234,10 @@ class Reporter {
console.log(padLines(stack, 4));
}
}
if (testRun.output) {
const output = testRun.output();
if (output.length) {
console.log(' Output:');
for (const line of testRun.output)
for (const line of output)
console.log(' ' + line);
}
}
Expand Down
Loading