diff --git a/packages/assert/__tests__/assert.test.ts b/packages/assert/__tests__/assert.test.ts index d096c3b7..d9353463 100644 --- a/packages/assert/__tests__/assert.test.ts +++ b/packages/assert/__tests__/assert.test.ts @@ -5,12 +5,13 @@ * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -import { assertAccessible, getViolationsJSDOM } from '../src/assert'; -import { base, extended, defaultPriority, getA11yConfig } from '@sa11y/preset-rules'; +import { assertAccessible, getA11yResultsJSDOM } from '../src/assert'; +import { base, extended, defaultPriority, getA11yConfig, adaptA11yConfigIncompleteResults } from '@sa11y/preset-rules'; import { a11yIssuesCount, audioURL, beforeEachSetup, + domWithA11yIncompleteIssues, domWithA11yIssues, domWithNoA11yIssues, shadowDomID, @@ -69,10 +70,16 @@ describe('assertAccessible API', () => { it.each([base, extended])('should throw no errors for dom with no a11y issues with config %#', async (config) => { document.body.innerHTML = domWithNoA11yIssues; - expect(async () => await getViolationsJSDOM(document, config)).toHaveLength(0); + expect(async () => await getA11yResultsJSDOM(document, config, false)).toHaveLength(0); await assertAccessible(document, config); // No error thrown }); + it('should throw errors for dom with incomplete issues with base config', async () => { + document.body.innerHTML = domWithA11yIncompleteIssues; + const config = adaptA11yConfigIncompleteResults(base); + expect(await getA11yResultsJSDOM(document, config, true)).toHaveLength(1); + }); + it.each([ // DOM to test, expected assertions, expected a11y violations [domWithNoA11yIssues, 2, 0], @@ -82,7 +89,7 @@ describe('assertAccessible API', () => { async (testDOM: string, expectedAssertions: number, expectedViolations: number) => { document.body.innerHTML = testDOM; expect.assertions(expectedAssertions); - await expect(getViolationsJSDOM()).resolves.toHaveLength(expectedViolations); + await expect(getA11yResultsJSDOM()).resolves.toHaveLength(expectedViolations); if (expectedViolations > 0) { // eslint-disable-next-line jest/no-conditional-expect await expect(assertAccessible()).rejects.toThrow(`${expectedViolations} Accessibility issues found`); diff --git a/packages/assert/src/assert.ts b/packages/assert/src/assert.ts index 24730c27..aa3ce0e6 100644 --- a/packages/assert/src/assert.ts +++ b/packages/assert/src/assert.ts @@ -8,7 +8,7 @@ import * as axe from 'axe-core'; import { defaultRuleset } from '@sa11y/preset-rules'; import { A11yError, exceptionListFilterSelectorKeywords } from '@sa11y/format'; -import { A11yConfig, AxeResults, getViolations } from '@sa11y/common'; +import { A11yConfig, AxeResults, getViolations, getIncomplete } from '@sa11y/common'; /** * Context that can be checked for accessibility: Document, Node or CSS selector. @@ -16,10 +16,36 @@ import { A11yConfig, AxeResults, getViolations } from '@sa11y/common'; */ export type A11yCheckableContext = Document | Node | string; +export async function getA11yResultsJSDOM( + context: A11yCheckableContext = document, + rules: A11yConfig = defaultRuleset, + enableIncompleteResults = false +): Promise { + return enableIncompleteResults ? getIncompleteJSDOM(context, rules) : getViolationsJSDOM(context, rules); +} + +/** + * Get list of a11y issues incomplete for given element and ruleset + * @param context - DOM or HTML Node to be tested for accessibility + * @param rules - A11yConfig preset rule to use, defaults to `base` ruleset + * @param reportType - Type of report ('violations' or 'incomplete') + * @returns {@link AxeResults} - list of accessibility issues found + */ +export async function getIncompleteJSDOM( + context: A11yCheckableContext = document, + rules: A11yConfig = defaultRuleset +): Promise { + return await getIncomplete(async () => { + const results = await axe.run(context as axe.ElementContext, rules as axe.RunOptions); + return results.incomplete; + }); +} + /** - * Get list of a11y violations for given element and ruleset + * Get list of a11y issues violations for given element and ruleset * @param context - DOM or HTML Node to be tested for accessibility * @param rules - A11yConfig preset rule to use, defaults to `base` ruleset + * @param reportType - Type of report ('violations' or 'incomplete') * @returns {@link AxeResults} - list of accessibility issues found */ export async function getViolationsJSDOM( diff --git a/packages/browser-lib/src/index.ts b/packages/browser-lib/src/index.ts index 70add7f8..1f3e1bab 100644 --- a/packages/browser-lib/src/index.ts +++ b/packages/browser-lib/src/index.ts @@ -8,35 +8,49 @@ import * as axe from 'axe-core'; import { exceptionListFilter, appendWcag } from '@sa11y/format'; import { registerCustomRules } from '@sa11y/common'; -import { defaultRuleset, changesData, rulesData, checkData } from '@sa11y/preset-rules'; -export { base, extended, full } from '@sa11y/preset-rules'; +import { + defaultRuleset, + changesData, + rulesData, + checkData, + adaptA11yConfigIncompleteResults, +} from '@sa11y/preset-rules'; +export { base, extended, full, adaptA11yConfigIncompleteResults } from '@sa11y/preset-rules'; export const namespace = 'sa11y'; /** * Wrapper function to check accessibility to be invoked after the sa11y minified JS file - * is injected into the browser - * @param scope - Scope of the analysis as {@link A11yCheckableContext} - * @param rules - preset sa11y rules defaulting to {@link base} - * @param exceptionList - mapping of rule to css selectors to be filtered out using {@link exceptionListFilter} + * is injected into the browser. + * + * @param scope - Scope of the analysis, defaults to the document. + * @param rules - Preset sa11y rules, defaults to {@link base}. + * @param exceptionList - Mapping of rule to CSS selectors to be filtered out using {@link exceptionListFilter}. + * @param addWcagInfo - Flag to add WCAG information to the results, defaults to false. + * @param enableIncompleteResults - Flag to include incomplete results in the analysis, defaults to false. */ // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types export async function checkAccessibility( scope = document, rules = defaultRuleset, exceptionList = {}, - addWcagInfo = true + addWcagInfo = true, + reportType: 'violations' | 'incomplete' = 'violations' ) { - // TODO (debug): adding type annotations to args, return type results in error: - // "[!] Error: Unexpected token" in both rollup-plugin-typescript2 and @rollup/plugin-typescript - // Compiling the index.ts file with tsc and using the dist/index.js file results - // in error when importing the "namespace" var. This would probably be easier to fix - // which could then result in getting rid of the rollup typescript plugins. + // TODO (debug): Adding type annotations to arguments and return type results in error: + // "[!] Error: Unexpected token" in both rollup-plugin-typescript2 and @rollup/plugin-typescript. + // Compiling index.ts with tsc and using the dist/index.js file results in an error when importing + // the "namespace" variable. This would probably be easier to fix, potentially allowing us to + // remove the rollup TypeScript plugins. // To register custom rules registerCustomRules(changesData, rulesData, checkData); + // Adapt rules if incomplete results are enabled + if (reportType === 'incomplete') { + rules = adaptA11yConfigIncompleteResults(rules); + } const results = await axe.run(scope || document, rules); - const filteredResults = exceptionListFilter(results.violations, exceptionList); + const filteredResults = exceptionListFilter(results[reportType], exceptionList); if (addWcagInfo) { appendWcag(filteredResults); diff --git a/packages/common/src/axe.ts b/packages/common/src/axe.ts index ec040697..3e3db16e 100644 --- a/packages/common/src/axe.ts +++ b/packages/common/src/axe.ts @@ -6,13 +6,13 @@ */ import * as axe from 'axe-core'; -import { RuleMetadata } from 'axe-core'; +import { resultGroups, RuleMetadata } from 'axe-core'; export const axeRuntimeExceptionMsgPrefix = 'Error running accessibility checks using axe:'; export const axeVersion: string | undefined = axe.version; -export type AxeResults = axe.Result[]; +export type AxeResults = axe.Result[] | axeIncompleteResults[]; /** * Interface that represents a function that runs axe and returns violations @@ -20,6 +20,9 @@ export type AxeResults = axe.Result[]; interface AxeRunner { (): Promise; } +export interface axeIncompleteResults extends axe.Result { + message?: string; +} /** * A11yConfig defines options to run accessibility checks using axe specifying list of rules to test @@ -29,21 +32,37 @@ export interface A11yConfig extends axe.RunOptions { type: 'rule'; values: string[]; }; - resultTypes: ['violations']; + resultTypes: resultGroups[]; } /** - * Get violations by running axe with given function + * Get results by running axe with given function * @param axeRunner - function satisfying AxeRunner interface */ -export async function getViolations(axeRunner: AxeRunner): Promise { - let violations; +export async function getA11yResults(axeRunner: AxeRunner): Promise { + let results; try { - violations = await axeRunner(); + results = await axeRunner(); } catch (e) { throw new Error(`${axeRuntimeExceptionMsgPrefix} ${(e as Error).message}`); } - return violations; + return results; +} + +/** + * Get incomplete by running axe with given function + * @param axeRunner - function satisfying AxeRunner interface + */ +export async function getIncomplete(axeRunner: AxeRunner): Promise { + return getA11yResults(axeRunner); +} + +/** + * Get violations by running axe with given function + * @param axeRunner - function satisfying AxeRunner interface + */ +export async function getViolations(axeRunner: AxeRunner): Promise { + return getA11yResults(axeRunner); } /** diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index 93966610..388e0a3f 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -5,7 +5,16 @@ * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -export { A11yConfig, AxeResults, axeRuntimeExceptionMsgPrefix, axeVersion, getAxeRules, getViolations } from './axe'; +export { + A11yConfig, + AxeResults, + axeIncompleteResults, + axeRuntimeExceptionMsgPrefix, + axeVersion, + getAxeRules, + getViolations, + getIncomplete, +} from './axe'; export { WdioAssertFunction, WdioOptions } from './wdio'; export { errMsgHeader, ExceptionList } from './format'; export { log, useFilesToBeExempted, useCustomRules, processFiles, registerCustomRules } from './helpers'; diff --git a/packages/format/__tests__/result.test.ts b/packages/format/__tests__/result.test.ts index 7b91b199..8f840888 100644 --- a/packages/format/__tests__/result.test.ts +++ b/packages/format/__tests__/result.test.ts @@ -123,6 +123,15 @@ describe('a11y result', () => { expect(A11yResults.add(a11yResults, key)).toHaveLength(a11yResults.length); expect(A11yResults.add(a11yResults, key)).toHaveLength(0); }); + it('should handle empty nodes violations', () => { + const violationWithoutNodes: AxeResults = []; + violations.forEach((violation) => { + violation.nodes = []; + violationWithoutNodes.push(violation); + }); + a11yResults = A11yResults.convert(violationWithoutNodes); + expect(a11yResults).toHaveLength(violationWithoutNodes.length); + }); }); describe('appendWcag', () => { diff --git a/packages/format/src/result.ts b/packages/format/src/result.ts index 70ec38c3..6878adc1 100644 --- a/packages/format/src/result.ts +++ b/packages/format/src/result.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -import { AxeResults } from '@sa11y/common'; +import { AxeResults, axeIncompleteResults } from '@sa11y/common'; import { NodeResult, Result, CheckResult } from 'axe-core'; import { priorities, wcagLevels, WcagMetadata } from '@sa11y/preset-rules'; @@ -61,6 +61,17 @@ export class A11yResults { */ static convert(violations: AxeResults): A11yResult[] { return A11yResults.sort(violations).flatMap((violation) => { + if (!violation.nodes?.length) { + // Handle case where nodes are empty by creating a default A11yResult + const emptyNodeResult: NodeResult = { + html: '', + target: [], + any: [], + all: [], + none: [], + }; + return [new A11yResult(violation, emptyNodeResult)]; + } return violation.nodes.map((node) => { return new A11yResult(violation, node); }); @@ -92,8 +103,9 @@ export class A11yResult { public readonly relatedNodeAll: string; public readonly relatedNodeNone: string; private readonly wcagData: WcagMetadata; // Used to sort results + public readonly message: string | undefined; - constructor(violation: Result, node: NodeResult) { + constructor(violation: Result | axeIncompleteResults, node: NodeResult) { this.id = violation.id; this.description = violation.description; this.wcagData = new WcagMetadata(violation); @@ -112,6 +124,7 @@ export class A11yResult { this.relatedNodeAny = this.formatRelatedNodes(node.any); this.relatedNodeAll = this.formatRelatedNodes(node.all); this.relatedNodeNone = this.formatRelatedNodes(node.none); + if ('message' in violation) this.message = violation?.message; } /** diff --git a/packages/jest/__tests__/__snapshots__/groupViolationResultsProcessor.test.ts.snap b/packages/jest/__tests__/__snapshots__/groupViolationResultsProcessor.test.ts.snap index fbfc83f7..4e31e4ed 100644 --- a/packages/jest/__tests__/__snapshots__/groupViolationResultsProcessor.test.ts.snap +++ b/packages/jest/__tests__/__snapshots__/groupViolationResultsProcessor.test.ts.snap @@ -1,5 +1,36 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`Group Violation Results Processor should process error Elements as expected 1`] = ` +" (1) - HTML element :
Click me
+ - CSS selector(s) : .button + - HTML Tag Hierarchy : body > div.button + - Error Message (Needs Review) : Ensure the element has a valid interactive role and behavior. + - More Info: To solve the problem, you need to fix at least (1) of the following: +role button is interactive + - Related Nodes: +none + - And fix the following: +element should be focusable and have a click handler + - Related Nodes: +none + - And fix the following: +no color contrast issues + - Related Nodes: +none + (2) - HTML element : \\"\\" + - CSS selector(s) : img + - HTML Tag Hierarchy : body > img + - Error Message (Needs Review) : Add an appropriate alt attribute describing the image. + - More Info: To solve the problem, you need to fix one of the following: +image elements must have an alt attribute + - Related Nodes: +none + - And fix the following: +no missing alt attribute allowed + - Related Nodes: +none" +`; + exports[`Group Violation Results Processor should process test results as expected 1`] = ` Object { "numFailedTestSuites": 1, diff --git a/packages/jest/__tests__/automatic.test.ts b/packages/jest/__tests__/automatic.test.ts index 22b699b4..e4cfb1ad 100644 --- a/packages/jest/__tests__/automatic.test.ts +++ b/packages/jest/__tests__/automatic.test.ts @@ -16,6 +16,7 @@ import { domWithDescendancyA11yIssues, customRulesFilePath, domWithA11yCustomIssues, + domWithA11yIncompleteIssues, } from '@sa11y/test-utils'; import * as Sa11yCommon from '@sa11y/common'; import { expect, jest } from '@jest/globals'; @@ -208,6 +209,16 @@ describe('automatic checks call', () => { await expect(automaticCheck({ filesFilter: nonExistentFilePaths })).rejects.toThrow(); }); + it('should incomplete rules', async () => { + setup(); + document.body.innerHTML = domWithA11yIncompleteIssues; + process.env.SA11Y_CUSTOM_RULES = customRulesFilePath; + await expect( + automaticCheck({ cleanupAfterEach: true, enableIncompleteResults: true, consolidateResults: true }) + ).rejects.toThrow('1 Accessibility'); + delete process.env.SA11Y_CUSTOM_RULES; + }); + it('should take only custom rules if specified/testing for new rule', async () => { setup(); document.body.innerHTML = domWithA11yCustomIssues; diff --git a/packages/jest/__tests__/groupViolationResultsProcessor.test.ts b/packages/jest/__tests__/groupViolationResultsProcessor.test.ts index f645c622..33c56b0c 100644 --- a/packages/jest/__tests__/groupViolationResultsProcessor.test.ts +++ b/packages/jest/__tests__/groupViolationResultsProcessor.test.ts @@ -10,7 +10,12 @@ import { A11yError, A11yResult } from '@sa11y/format'; import { getA11yError } from '@sa11y/format/__tests__/format.test'; import { domWithVisualA11yIssues } from '@sa11y/test-utils'; import { expect } from '@jest/globals'; -import { resultsProcessor, resultsProcessorManualChecks } from '../src/groupViolationResultsProcessor'; +import { + resultsProcessor, + resultsProcessorManualChecks, + ErrorElement, + createA11yErrorElements, +} from '../src/groupViolationResultsProcessor'; const a11yResults: A11yResult[] = []; const aggregatedResults = makeEmptyAggregatedTestResult(); @@ -83,4 +88,36 @@ describe('Group Violation Results Processor', () => { expect(processedResults).not.toEqual(aggregatedResults); expect(processedResults.numFailedTestSuites).toBe(1); }); + + it('should process error Elements as expected', () => { + const errorElements: ErrorElement[] = [ + { + html: '
Click me
', + selectors: '.button', + hierarchy: 'body > div.button', + any: 'role button is interactive', + all: 'element should be focusable and have a click handler', + none: 'no color contrast issues', + relatedNodeAny: 'none', + relatedNodeAll: 'none', + relatedNodeNone: 'none', + message: 'Ensure the element has a valid interactive role and behavior.', + }, + { + html: '', + selectors: 'img', + hierarchy: 'body > img', + all: 'image elements must have an alt attribute', + none: 'no missing alt attribute allowed', + relatedNodeAny: 'none', + relatedNodeAll: 'none', + relatedNodeNone: 'none', + message: 'Add an appropriate alt attribute describing the image.', + any: '', + }, + ]; + + const createdA11yErrorElements = createA11yErrorElements(errorElements); + expect(createdA11yErrorElements).toMatchSnapshot(); + }); }); diff --git a/packages/jest/__tests__/setup.test.ts b/packages/jest/__tests__/setup.test.ts index e4f2cc15..5ee48403 100644 --- a/packages/jest/__tests__/setup.test.ts +++ b/packages/jest/__tests__/setup.test.ts @@ -5,8 +5,7 @@ * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -import { adaptA11yConfig, adaptA11yConfigCustomRules, registerSa11yMatcher } from '../src/setup'; -import { base, extended } from '@sa11y/preset-rules'; +import { registerSa11yMatcher } from '../src/setup'; import { expect, jest } from '@jest/globals'; describe('jest setup', () => { @@ -15,22 +14,6 @@ describe('jest setup', () => { expect(expect['toBeAccessible']).toBeDefined(); }); - it.each([extended, base])('should customize %s preset-rule as expected', (config) => { - expect(config.runOnly.values).toContain('color-contrast'); - expect(adaptA11yConfig(config).runOnly.values).not.toContain('color-contrast'); - // original ruleset is not modified - expect(config.runOnly.values).toContain('color-contrast'); - }); - it('should customize preset-rule as expected for custom rules', () => { - expect(base.runOnly.values).toContain('color-contrast'); - const rules = adaptA11yConfigCustomRules(base, ['rule1', 'rule2']).runOnly.values; - expect(rules).toContain('rule1'); - expect(rules).toContain('rule2'); - - // original ruleset is not modified - expect(base.runOnly.values).toContain('color-contrast'); - }); - /* Skipped: Difficult to mock the global "expect" when we are `import {expect} from '@jest/globals'` */ it.skip('should throw error when global expect is undefined', () => { const globalExpect = expect as jest.Expect; diff --git a/packages/jest/src/automatic.ts b/packages/jest/src/automatic.ts index eb49d471..de811bea 100644 --- a/packages/jest/src/automatic.ts +++ b/packages/jest/src/automatic.ts @@ -6,12 +6,16 @@ */ import { AxeResults, log, useCustomRules } from '@sa11y/common'; -import { getViolationsJSDOM } from '@sa11y/assert'; +import { getA11yResultsJSDOM } from '@sa11y/assert'; import { A11yError, exceptionListFilterSelectorKeywords } from '@sa11y/format'; import { isTestUsingFakeTimer } from './matcher'; import { expect } from '@jest/globals'; -import { adaptA11yConfig, adaptA11yConfigCustomRules } from './setup'; -import { defaultRuleset } from '@sa11y/preset-rules'; +import { + defaultRuleset, + adaptA11yConfig, + adaptA11yConfigCustomRules, + adaptA11yConfigIncompleteResults, +} from '@sa11y/preset-rules'; /** * Options for Automatic checks to be passed to {@link registerSa11yAutomaticChecks} @@ -25,6 +29,7 @@ export type AutoCheckOpts = { // List of test file paths (as regex) to filter for automatic checks filesFilter?: string[]; runDOMMutationObserver?: boolean; + enableIncompleteResults?: boolean; }; /** @@ -36,6 +41,7 @@ const defaultAutoCheckOpts: AutoCheckOpts = { consolidateResults: true, filesFilter: [], runDOMMutationObserver: false, + enableIncompleteResults: false, }; let originalDocumentBodyHtml: string | null = null; @@ -78,7 +84,13 @@ export async function automaticCheck(opts: AutoCheckOpts = defaultAutoCheckOpts) return; } - let violations: AxeResults = []; + const customRules = useCustomRules(); + let config = + customRules.length === 0 + ? adaptA11yConfig(defaultRuleset) + : adaptA11yConfigCustomRules(defaultRuleset, customRules); + if (opts.enableIncompleteResults) config = adaptA11yConfigIncompleteResults(config); + let a11yResults: AxeResults = []; const currentDocumentHtml = document.body.innerHTML; if (originalDocumentBodyHtml) { document.body.innerHTML = originalDocumentBodyHtml; @@ -86,7 +98,6 @@ export async function automaticCheck(opts: AutoCheckOpts = defaultAutoCheckOpts) // Create a DOM walker filtering only elements (skipping text, comment nodes etc) const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT); let currNode = walker.firstChild(); - const customRules = useCustomRules(); try { if (!opts.runDOMMutationObserver) { while (currNode !== null) { @@ -97,38 +108,19 @@ export async function automaticCheck(opts: AutoCheckOpts = defaultAutoCheckOpts) // : ${testPath}` // ); // W-10004832 - Exclude descendancy based rules from automatic checks - if (customRules.length === 0) - violations.push(...(await getViolationsJSDOM(currNode, adaptA11yConfig(defaultRuleset)))); - else - violations.push( - ...(await getViolationsJSDOM(currNode, adaptA11yConfigCustomRules(defaultRuleset, customRules))) - ); + a11yResults.push(...(await getA11yResultsJSDOM(currNode, config, opts.enableIncompleteResults))); currNode = walker.nextSibling(); } } else { - if (customRules.length === 0) - violations.push(...(await getViolationsJSDOM(document.body, adaptA11yConfig(defaultRuleset)))); - else - violations.push( - ...(await getViolationsJSDOM( - document.body, - adaptA11yConfigCustomRules(defaultRuleset, customRules) - )) - ); + a11yResults.push(...(await getA11yResultsJSDOM(document.body, config, opts.enableIncompleteResults))); document.body.innerHTML = ''; // loop mutated nodes for await (const mutatedNode of mutatedNodes) { if (mutatedNode) { document.body.innerHTML = mutatedNode; - if (customRules.length === 0) - violations.push(...(await getViolationsJSDOM(document.body, adaptA11yConfig(defaultRuleset)))); - else - violations.push( - ...(await getViolationsJSDOM( - document.body, - adaptA11yConfigCustomRules(defaultRuleset, customRules) - )) - ); + a11yResults.push( + ...(await getA11yResultsJSDOM(document.body, config, opts.enableIncompleteResults)) + ); } } } @@ -143,12 +135,12 @@ export async function automaticCheck(opts: AutoCheckOpts = defaultAutoCheckOpts) // Will this affect all errors globally? // Error.stackTraceLimit = 0; if (process.env.SELECTOR_FILTER_KEYWORDS) { - violations = exceptionListFilterSelectorKeywords( - violations, + a11yResults = exceptionListFilterSelectorKeywords( + a11yResults, process.env.SELECTOR_FILTER_KEYWORDS.split(',') ); } - A11yError.checkAndThrow(violations, { deduplicate: opts.consolidateResults }); + A11yError.checkAndThrow(a11yResults, { deduplicate: opts.consolidateResults }); } } diff --git a/packages/jest/src/groupViolationResultsProcessor.ts b/packages/jest/src/groupViolationResultsProcessor.ts index f9c40420..6df59296 100644 --- a/packages/jest/src/groupViolationResultsProcessor.ts +++ b/packages/jest/src/groupViolationResultsProcessor.ts @@ -19,7 +19,7 @@ interface FailureMatcherDetail { }; } -type ErrorElement = { +export type ErrorElement = { html: string; selectors: string; hierarchy: string; @@ -29,6 +29,7 @@ type ErrorElement = { relatedNodeAny: string; relatedNodeAll: string; relatedNodeNone: string; + message?: string; }; type A11yViolation = { @@ -52,7 +53,7 @@ const axeMessages = { /** * Create a test failure html elements array grouped by rule violation */ -function createA11yErrorElements(errorElements: ErrorElement[]) { +export function createA11yErrorElements(errorElements: ErrorElement[]) { const a11yErrorElements: string[] = []; let onlyOneSummary = false; @@ -66,6 +67,8 @@ function createA11yErrorElements(errorElements: ErrorElement[]) { .replace(/>/, '>')}`; errorMessage += `${formatForAxeMessage}- CSS selector(s) : ${errorElement.selectors.replace(/>/, '>')}`; errorMessage += `${formatForAxeMessage}- HTML Tag Hierarchy : ${errorElement.hierarchy}`; + if (errorElement.message) + errorMessage += `${formatForAxeMessage}- Error Message (Needs Review) : ${errorElement.message}`; const isAny = errorElement.any?.length > 0; const isAll = errorElement.all?.length > 0; const isNone = errorElement.none?.length > 0; @@ -147,6 +150,7 @@ function processA11yDetailsAndMessages(error: A11yError, a11yFailureMessages: st relatedNodeAny: a11yResult.relatedNodeAny, relatedNodeAll: a11yResult.relatedNodeAll, relatedNodeNone: a11yResult.relatedNodeNone, + message: a11yResult?.message, }); }); @@ -250,6 +254,7 @@ export function resultsProcessorManualChecks(results: AggregatedResult): Aggrega const exportedFunctions = { resultsProcessor, resultsProcessorManualChecks, + createA11yErrorElements, }; module.exports = exportedFunctions; diff --git a/packages/jest/src/matcher.ts b/packages/jest/src/matcher.ts index 61b5bcbe..3fdc7e17 100644 --- a/packages/jest/src/matcher.ts +++ b/packages/jest/src/matcher.ts @@ -6,10 +6,9 @@ */ import { matcherHint, printReceived } from 'jest-matcher-utils'; -import { adaptA11yConfig } from './setup'; import { A11yCheckableContext, assertAccessible } from '@sa11y/assert'; import { A11yError, Options } from '@sa11y/format'; -import { defaultRuleset } from '@sa11y/preset-rules'; +import { defaultRuleset, adaptA11yConfig } from '@sa11y/preset-rules'; import { A11yConfig } from '@sa11y/common'; // Type def for custom jest a11y matchers diff --git a/packages/jest/src/setup.ts b/packages/jest/src/setup.ts index d08e343e..4ab49ab2 100644 --- a/packages/jest/src/setup.ts +++ b/packages/jest/src/setup.ts @@ -6,7 +6,7 @@ */ import { toBeAccessible } from './matcher'; -import { A11yConfig, useFilesToBeExempted, registerCustomRules } from '@sa11y/common'; +import { useFilesToBeExempted, registerCustomRules } from '@sa11y/common'; import { AutoCheckOpts, registerSa11yAutomaticChecks, @@ -16,25 +16,6 @@ import { import { expect } from '@jest/globals'; import { changesData, rulesData, checkData } from '@sa11y/preset-rules'; -export const disabledRules = [ - // Descendancy checks that would fail at unit/component level, but pass at page level - 'aria-required-children', - 'aria-required-parent', - 'dlitem', - 'definition-list', - 'list', - 'listitem', - 'landmark-one-main', - // color-contrast doesn't work for JSDOM and might affect performance - // https://github.com/dequelabs/axe-core/issues/595 - // https://github.com/dequelabs/axe-core/blob/develop/doc/examples/jsdom/test/a11y.js - 'color-contrast', - // audio, video elements are stubbed out in JSDOM - // https://github.com/jsdom/jsdom/issues/2155 - 'audio-caption', - 'video-caption', -]; - /** * Options to be passed on to {@link setup} */ @@ -55,6 +36,7 @@ const defaultSa11yOpts: Sa11yOpts = { cleanupAfterEach: false, consolidateResults: false, runDOMMutationObserver: false, + enableIncompleteResults: false, }, }; @@ -130,6 +112,7 @@ export function setup(opts: Sa11yOpts = defaultSa11yOpts): void { // set the flag to true to run sa11y in DOM Mutation Observer mode autoCheckOpts.runDOMMutationObserver ||= !!process.env.SA11Y_ENABLE_DOM_MUTATION_OBSERVER; + autoCheckOpts.enableIncompleteResults ||= !!process.env.SA11Y_ENABLE_INCOMPLETE_RESULTS; registerSa11yAutomaticChecks(autoCheckOpts); } /** @@ -146,19 +129,3 @@ export function registerSa11yMatcher(): void { ); } } -/** - * Customize sa11y preset rules specific to JSDOM - */ -export function adaptA11yConfigCustomRules(config: A11yConfig, customRules: string[]): A11yConfig { - const adaptedConfig = JSON.parse(JSON.stringify(config)) as A11yConfig; - adaptedConfig.runOnly.values = customRules; - adaptedConfig.ancestry = true; - return adaptedConfig; -} -export function adaptA11yConfig(config: A11yConfig, filterRules = disabledRules): A11yConfig { - // TODO (refactor): Move into preset-rules pkg as a generic rules filter util - const adaptedConfig = JSON.parse(JSON.stringify(config)) as A11yConfig; - adaptedConfig.runOnly.values = config.runOnly.values.filter((rule) => !filterRules.includes(rule)); - adaptedConfig.ancestry = true; - return adaptedConfig; -} diff --git a/packages/preset-rules/__tests__/rules.test.ts b/packages/preset-rules/__tests__/rules.test.ts index db1685f1..48d020a9 100644 --- a/packages/preset-rules/__tests__/rules.test.ts +++ b/packages/preset-rules/__tests__/rules.test.ts @@ -9,7 +9,15 @@ import { base, full, extended, defaultRuleset, excludedRules, getDefaultRuleset import { baseRulesInfo } from '../src/base'; import { extendedRulesInfo } from '../src/extended'; import { getRulesDoc } from '../src/docgen'; -import { filterRulesByPriority, getPriorityFilter, priorities, RuleInfo } from '../src/rules'; +import { + filterRulesByPriority, + getPriorityFilter, + priorities, + RuleInfo, + adaptA11yConfig, + adaptA11yConfigCustomRules, + adaptA11yConfigIncompleteResults, +} from '../src/rules'; import { axeVersion } from '@sa11y/common'; import * as axe from 'axe-core'; import * as fs from 'fs'; @@ -134,3 +142,28 @@ describe('preset-rules priority config', () => { expect(getPriorityFilter()).toEqual(priority); }); }); + +describe('config adapt functions', () => { + it.each([extended, base])('should customize %s preset-rule as expected', (config) => { + expect(config.runOnly.values).toContain('color-contrast'); + expect(adaptA11yConfig(config).runOnly.values).not.toContain('color-contrast'); + // original ruleset is not modified + expect(config.runOnly.values).toContain('color-contrast'); + }); + + it('should customize preset-rule as expected for custom rules', () => { + expect(base.runOnly.values).toContain('color-contrast'); + const rules = adaptA11yConfigCustomRules(base, ['rule1', 'rule2']).runOnly.values; + expect(rules).toContain('rule1'); + expect(rules).toContain('rule2'); + + // original ruleset is not modified + expect(base.runOnly.values).toContain('color-contrast'); + }); + + it('should customize config as expected for incomplete results', () => { + expect(base.reporter).toBe('no-passes'); + const changedReporter = adaptA11yConfigIncompleteResults(base).reporter; + expect(changedReporter).toContain('v2'); + }); +}); diff --git a/packages/preset-rules/src/index.ts b/packages/preset-rules/src/index.ts index 723756c6..d7f5dc52 100644 --- a/packages/preset-rules/src/index.ts +++ b/packages/preset-rules/src/index.ts @@ -5,7 +5,16 @@ * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause */ -export { defaultPriority, priorities, wcagLevels, getA11yConfig, RuleInfo } from './rules'; +export { + defaultPriority, + priorities, + wcagLevels, + adaptA11yConfig, + adaptA11yConfigCustomRules, + adaptA11yConfigIncompleteResults, + getA11yConfig, + RuleInfo, +} from './rules'; export { defaultRuleset, getDefaultRuleset } from './config'; export { extended } from './extended'; export { base } from './base'; diff --git a/packages/preset-rules/src/rules.ts b/packages/preset-rules/src/rules.ts index 9e8e7547..b3885ba2 100644 --- a/packages/preset-rules/src/rules.ts +++ b/packages/preset-rules/src/rules.ts @@ -15,6 +15,25 @@ export type Priority = (typeof priorities)[number]; export const defaultPriority: Priority = 'P3'; export const defaultWcagVersion: WcagVersion = '2.1'; +export const disabledRules = [ + // Descendancy checks that would fail at unit/component level, but pass at page level + 'aria-required-children', + 'aria-required-parent', + 'dlitem', + 'definition-list', + 'list', + 'listitem', + 'landmark-one-main', + // color-contrast doesn't work for JSDOM and might affect performance + // https://github.com/dequelabs/axe-core/issues/595 + // https://github.com/dequelabs/axe-core/blob/develop/doc/examples/jsdom/test/a11y.js + 'color-contrast', + // audio, video elements are stubbed out in JSDOM + // https://github.com/jsdom/jsdom/issues/2155 + 'audio-caption', + 'video-caption', +]; + /** * Metadata about rules such as Priority and WCAG SC (overriding values from axe tags) */ @@ -63,6 +82,36 @@ export function filterRulesByPriority(rules: RuleInfo, priority: Priority = ''): return ruleIDs.sort(); } +/** + * Customize sa11y preset rules specific to JSDOM + */ +export function adaptA11yConfigCustomRules(config: A11yConfig, customRules: string[]): A11yConfig { + const adaptedConfig = JSON.parse(JSON.stringify(config)) as A11yConfig; + adaptedConfig.runOnly.values = customRules; + adaptedConfig.ancestry = true; + return adaptedConfig; +} + +/** + * Adapt a11y config to report only incomplete results + */ +export function adaptA11yConfigIncompleteResults(config: A11yConfig): A11yConfig { + const adaptedConfig = JSON.parse(JSON.stringify(config)) as A11yConfig; + adaptedConfig.reporter = 'v2'; + adaptedConfig.resultTypes = ['incomplete']; + return adaptedConfig; +} + +/** + * Adapt a11y config by filtering out disabled rules + */ +export function adaptA11yConfig(config: A11yConfig, filterRules = disabledRules): A11yConfig { + const adaptedConfig = JSON.parse(JSON.stringify(config)) as A11yConfig; + adaptedConfig.runOnly.values = config.runOnly.values.filter((rule) => !filterRules.includes(rule)); + adaptedConfig.ancestry = true; + return adaptedConfig; +} + /** * Returns config to be used in axe.run() with given rules * Ref: https://github.com/dequelabs/axe-core/blob/develop/doc/API.md#options-parameter diff --git a/packages/test-utils/__data__/a11yIncompleteIssues.html b/packages/test-utils/__data__/a11yIncompleteIssues.html new file mode 100644 index 00000000..15643fd7 --- /dev/null +++ b/packages/test-utils/__data__/a11yIncompleteIssues.html @@ -0,0 +1,30 @@ + + + + + + Axe Needs Review Test + + + + + +

Axe Needs Review Test

+ +
+ Pass
+ Fail +
+ + diff --git a/packages/test-utils/__data__/sa11y-custom-rules.json b/packages/test-utils/__data__/sa11y-custom-rules.json index c7ebfc16..f3d462ee 100644 --- a/packages/test-utils/__data__/sa11y-custom-rules.json +++ b/packages/test-utils/__data__/sa11y-custom-rules.json @@ -1,3 +1,3 @@ { - "rules": ["sa11y-Keyboard"] + "rules": ["sa11y-Keyboard", "color-contrast"] } diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts index 3e862eb4..adf08710 100644 --- a/packages/test-utils/src/index.ts +++ b/packages/test-utils/src/index.ts @@ -23,5 +23,6 @@ export { videoURL, customRulesFilePath, domWithA11yCustomIssues, + domWithA11yIncompleteIssues, } from './test-data'; export { beforeEachSetup, cartesianProduct } from './utils'; diff --git a/packages/test-utils/src/test-data.ts b/packages/test-utils/src/test-data.ts index d1bb331b..1686fae9 100644 --- a/packages/test-utils/src/test-data.ts +++ b/packages/test-utils/src/test-data.ts @@ -15,11 +15,13 @@ export const domWithA11yIssuesBodyID = 'dom-with-issues'; const fileWithA11yIssues = path.resolve(dataDir, 'a11yIssues.html'); export const customRulesFilePath = path.resolve(dataDir, 'sa11y-custom-rules.json'); export const domWithA11yCustomIssuesPath = path.resolve(dataDir, 'a11yCustomIssues.html'); +export const domWithA11yIncompleteIssuesPath = path.resolve(dataDir, 'a11yIncompleteIssues.html'); const fileWithDescendancyA11yIssues = path.resolve(dataDir, 'descendancyA11yIssues.html'); export const htmlFileWithA11yIssues = 'file:///' + fileWithA11yIssues; export const domWithA11yIssues = fs.readFileSync(fileWithA11yIssues).toString(); export const domWithA11yCustomIssues = fs.readFileSync(domWithA11yCustomIssuesPath).toString(); +export const domWithA11yIncompleteIssues = fs.readFileSync(domWithA11yIncompleteIssuesPath).toString(); export const domWithDescendancyA11yIssues = fs.readFileSync(fileWithDescendancyA11yIssues).toString(); export const a11yIssuesCount = 5;