feat: added functionality of incomplete resultType (#730)

* feat: added functionality of incomplete resultType

* Update packages/test-utils/__data__/a11yIncompleteIssues.html

Co-authored-by: Navateja Alagam <[email protected]>

* feat: changed the function defination of getViolationJSDOM

---------

Co-authored-by: Navateja Alagam <[email protected]>

packages/assert/__tests__/assert.test.ts

@@ -5,12 +5,13 @@
  * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 
-import { assertAccessible, getViolationsJSDOM } from '../src/assert';
-import { base, extended, defaultPriority, getA11yConfig } from '@sa11y/preset-rules';
+import { assertAccessible, getA11yResultsJSDOM } from '../src/assert';
+import { base, extended, defaultPriority, getA11yConfig, adaptA11yConfigIncompleteResults } from '@sa11y/preset-rules';
 import {
  a11yIssuesCount,
  audioURL,
  beforeEachSetup,
+ domWithA11yIncompleteIssues,
  domWithA11yIssues,
  domWithNoA11yIssues,
  shadowDomID,
@@ -69,10 +70,16 @@ describe('assertAccessible API', () => {
 
  it.each([base, extended])('should throw no errors for dom with no a11y issues with config %#', async (config) => {
  document.body.innerHTML = domWithNoA11yIssues;
- expect(async () => await getViolationsJSDOM(document, config)).toHaveLength(0);
+ expect(async () => await getA11yResultsJSDOM(document, config, false)).toHaveLength(0);
  await assertAccessible(document, config); // No error thrown
  });
 
+ it('should throw errors for dom with incomplete issues with base config', async () => {
+ document.body.innerHTML = domWithA11yIncompleteIssues;
+ const config = adaptA11yConfigIncompleteResults(base);
+ expect(await getA11yResultsJSDOM(document, config, true)).toHaveLength(1);
+ });
+
  it.each([
  // DOM to test, expected assertions, expected a11y violations
  [domWithNoA11yIssues, 2, 0],
@@ -82,7 +89,7 @@ describe('assertAccessible API', () => {
  async (testDOM: string, expectedAssertions: number, expectedViolations: number) => {
  document.body.innerHTML = testDOM;
  expect.assertions(expectedAssertions);
- await expect(getViolationsJSDOM()).resolves.toHaveLength(expectedViolations);
+ await expect(getA11yResultsJSDOM()).resolves.toHaveLength(expectedViolations);
  if (expectedViolations > 0) {
  // eslint-disable-next-line jest/no-conditional-expect
  await expect(assertAccessible()).rejects.toThrow(`${expectedViolations} Accessibility issues found`);

packages/assert/src/assert.ts

@@ -8,18 +8,44 @@
 import * as axe from 'axe-core';
 import { defaultRuleset } from '@sa11y/preset-rules';
 import { A11yError, exceptionListFilterSelectorKeywords } from '@sa11y/format';
-import { A11yConfig, AxeResults, getViolations } from '@sa11y/common';
+import { A11yConfig, AxeResults, getViolations, getIncomplete } from '@sa11y/common';
 
 /**
  * Context that can be checked for accessibility: Document, Node or CSS selector.
  * Limiting to subset of options supported by axe for ease of use and maintenance.
  */
 export type A11yCheckableContext = Document | Node | string;
 
+export async function getA11yResultsJSDOM(
+ context: A11yCheckableContext = document,
+ rules: A11yConfig = defaultRuleset,
+ enableIncompleteResults = false
+): Promise<AxeResults> {
+ return enableIncompleteResults ? getIncompleteJSDOM(context, rules) : getViolationsJSDOM(context, rules);
+}
+
+/**
+ * Get list of a11y issues incomplete for given element and ruleset
+ * @param context - DOM or HTML Node to be tested for accessibility
+ * @param rules - A11yConfig preset rule to use, defaults to `base` ruleset
+ * @param reportType - Type of report ('violations' or 'incomplete')
+ * @returns {@link AxeResults} - list of accessibility issues found
+ */
+export async function getIncompleteJSDOM(
+ context: A11yCheckableContext = document,
+ rules: A11yConfig = defaultRuleset
+): Promise<AxeResults> {
+ return await getIncomplete(async () => {
+ const results = await axe.run(context as axe.ElementContext, rules as axe.RunOptions);
+ return results.incomplete;
+ });
+}
+
 /**
- * Get list of a11y violations for given element and ruleset
+ * Get list of a11y issues violations for given element and ruleset
  * @param context - DOM or HTML Node to be tested for accessibility
  * @param rules - A11yConfig preset rule to use, defaults to `base` ruleset
+ * @param reportType - Type of report ('violations' or 'incomplete')
  * @returns {@link AxeResults} - list of accessibility issues found
  */
 export async function getViolationsJSDOM(

packages/browser-lib/src/index.ts

@@ -8,35 +8,49 @@
 import * as axe from 'axe-core';
 import { exceptionListFilter, appendWcag } from '@sa11y/format';
 import { registerCustomRules } from '@sa11y/common';
-import { defaultRuleset, changesData, rulesData, checkData } from '@sa11y/preset-rules';
-export { base, extended, full } from '@sa11y/preset-rules';
+import {
+ defaultRuleset,
+ changesData,
+ rulesData,
+ checkData,
+ adaptA11yConfigIncompleteResults,
+} from '@sa11y/preset-rules';
+export { base, extended, full, adaptA11yConfigIncompleteResults } from '@sa11y/preset-rules';
 export const namespace = 'sa11y';
 
 /**
  * Wrapper function to check accessibility to be invoked after the sa11y minified JS file
- * is injected into the browser
- * @param scope - Scope of the analysis as {@link A11yCheckableContext}
- * @param rules - preset sa11y rules defaulting to {@link base}
- * @param exceptionList - mapping of rule to css selectors to be filtered out using {@link exceptionListFilter}
+ * is injected into the browser.
+ *
+ * @param scope - Scope of the analysis, defaults to the document.
+ * @param rules - Preset sa11y rules, defaults to {@link base}.
+ * @param exceptionList - Mapping of rule to CSS selectors to be filtered out using {@link exceptionListFilter}.
+ * @param addWcagInfo - Flag to add WCAG information to the results, defaults to false.
+ * @param enableIncompleteResults - Flag to include incomplete results in the analysis, defaults to false.
  */
 // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
 export async function checkAccessibility(
  scope = document,
  rules = defaultRuleset,
  exceptionList = {},
- addWcagInfo = true
+ addWcagInfo = true,
+ reportType: 'violations' | 'incomplete' = 'violations'
 ) {
- // TODO (debug): adding type annotations to args, return type results in error:
- //  "[!] Error: Unexpected token" in both rollup-plugin-typescript2 and @rollup/plugin-typescript
- //  Compiling the index.ts file with tsc and using the dist/index.js file results
- //  in error when importing the "namespace" var. This would probably be easier to fix
- //  which could then result in getting rid of the rollup typescript plugins.
+ // TODO (debug): Adding type annotations to arguments and return type results in error:
+ // "[!] Error: Unexpected token" in both rollup-plugin-typescript2 and @rollup/plugin-typescript.
+ // Compiling index.ts with tsc and using the dist/index.js file results in an error when importing
+ // the "namespace" variable. This would probably be easier to fix, potentially allowing us to
+ // remove the rollup TypeScript plugins.
 
  // To register custom rules
  registerCustomRules(changesData, rulesData, checkData);
 
+ // Adapt rules if incomplete results are enabled
+ if (reportType === 'incomplete') {
+ rules = adaptA11yConfigIncompleteResults(rules);
+ }
  const results = await axe.run(scope || document, rules);
- const filteredResults = exceptionListFilter(results.violations, exceptionList);
+ const filteredResults = exceptionListFilter(results[reportType], exceptionList);
 
  if (addWcagInfo) {
  appendWcag(filteredResults);

packages/common/src/axe.ts

@@ -6,20 +6,23 @@
  */
 
 import * as axe from 'axe-core';
-import { RuleMetadata } from 'axe-core';
+import { resultGroups, RuleMetadata } from 'axe-core';
 
 export const axeRuntimeExceptionMsgPrefix = 'Error running accessibility checks using axe:';
 
 export const axeVersion: string | undefined = axe.version;
 
-export type AxeResults = axe.Result[];
+export type AxeResults = axe.Result[] | axeIncompleteResults[];
 
 /**
  * Interface that represents a function that runs axe and returns violations
  */
 interface AxeRunner {
  (): Promise<AxeResults>;
 }
+export interface axeIncompleteResults extends axe.Result {
+ message?: string;
+}
 
 /**
  * A11yConfig defines options to run accessibility checks using axe specifying list of rules to test
@@ -29,21 +32,37 @@ export interface A11yConfig extends axe.RunOptions {
  type: 'rule';
  values: string[];
  };
- resultTypes: ['violations'];
+ resultTypes: resultGroups[];
 }
 
 /**
- * Get violations by running axe with given function
+ * Get results by running axe with given function
  * @param axeRunner - function satisfying AxeRunner interface
  */
-export async function getViolations(axeRunner: AxeRunner): Promise<AxeResults> {
- let violations;
+export async function getA11yResults(axeRunner: AxeRunner): Promise<AxeResults> {
+ let results;
  try {
- violations = await axeRunner();
+ results = await axeRunner();
  } catch (e) {
  throw new Error(`${axeRuntimeExceptionMsgPrefix} ${(e as Error).message}`);
  }
- return violations;
+ return results;
+}
+
+/**
+ * Get incomplete by running axe with given function
+ * @param axeRunner - function satisfying AxeRunner interface
+ */
+export async function getIncomplete(axeRunner: AxeRunner): Promise<AxeResults> {
+ return getA11yResults(axeRunner);
+}
+
+/**
+ * Get violations by running axe with given function
+ * @param axeRunner - function satisfying AxeRunner interface
+ */
+export async function getViolations(axeRunner: AxeRunner): Promise<AxeResults> {
+ return getA11yResults(axeRunner);
 }
 
 /**

packages/common/src/index.ts

@@ -5,7 +5,16 @@
  * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
 
-export { A11yConfig, AxeResults, axeRuntimeExceptionMsgPrefix, axeVersion, getAxeRules, getViolations } from './axe';
+export {
+ A11yConfig,
+ AxeResults,
+ axeIncompleteResults,
+ axeRuntimeExceptionMsgPrefix,
+ axeVersion,
+ getAxeRules,
+ getViolations,
+ getIncomplete,
+} from './axe';
 export { WdioAssertFunction, WdioOptions } from './wdio';
 export { errMsgHeader, ExceptionList } from './format';
 export { log, useFilesToBeExempted, useCustomRules, processFiles, registerCustomRules } from './helpers';

packages/format/__tests__/result.test.ts

@@ -123,6 +123,15 @@ describe('a11y result', () => {
  expect(A11yResults.add(a11yResults, key)).toHaveLength(a11yResults.length);
  expect(A11yResults.add(a11yResults, key)).toHaveLength(0);
  });
+ it('should handle empty nodes violations', () => {
+ const violationWithoutNodes: AxeResults = [];
+ violations.forEach((violation) => {
+ violation.nodes = [];
+ violationWithoutNodes.push(violation);
+ });
+ a11yResults = A11yResults.convert(violationWithoutNodes);
+ expect(a11yResults).toHaveLength(violationWithoutNodes.length);
+ });
 });
 
 describe('appendWcag', () => {

packages/format/src/result.ts

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: BSD-3-Clause
  * For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
  */
-import { AxeResults } from '@sa11y/common';
+import { AxeResults, axeIncompleteResults } from '@sa11y/common';
 import { NodeResult, Result, CheckResult } from 'axe-core';
 import { priorities, wcagLevels, WcagMetadata } from '@sa11y/preset-rules';
 
@@ -61,6 +61,17 @@ export class A11yResults {
  */
  static convert(violations: AxeResults): A11yResult[] {
  return A11yResults.sort(violations).flatMap((violation) => {
+ if (!violation.nodes?.length) {
+ // Handle case where nodes are empty by creating a default A11yResult
+ const emptyNodeResult: NodeResult = {
+ html: '',
+ target: [],
+ any: [],
+ all: [],
+ none: [],
+ };
+ return [new A11yResult(violation, emptyNodeResult)];
+ }
  return violation.nodes.map((node) => {
  return new A11yResult(violation, node);
  });
@@ -92,8 +103,9 @@ export class A11yResult {
  public readonly relatedNodeAll: string;
  public readonly relatedNodeNone: string;
  private readonly wcagData: WcagMetadata; // Used to sort results
+ public readonly message: string | undefined;
 
- constructor(violation: Result, node: NodeResult) {
+ constructor(violation: Result | axeIncompleteResults, node: NodeResult) {
  this.id = violation.id;
  this.description = violation.description;
  this.wcagData = new WcagMetadata(violation);
@@ -112,6 +124,7 @@ export class A11yResult {
  this.relatedNodeAny = this.formatRelatedNodes(node.any);
  this.relatedNodeAll = this.formatRelatedNodes(node.all);
  this.relatedNodeNone = this.formatRelatedNodes(node.none);
+ if ('message' in violation) this.message = violation?.message;
  }
 
  /**

packages/jest/__tests__/automatic.test.ts

@@ -16,6 +16,7 @@ import {
  domWithDescendancyA11yIssues,
  customRulesFilePath,
  domWithA11yCustomIssues,
+ domWithA11yIncompleteIssues,
 } from '@sa11y/test-utils';
 import * as Sa11yCommon from '@sa11y/common';
 import { expect, jest } from '@jest/globals';
@@ -208,6 +209,16 @@ describe('automatic checks call', () => {
  await expect(automaticCheck({ filesFilter: nonExistentFilePaths })).rejects.toThrow();
  });
 
+ it('should incomplete rules', async () => {
+ setup();
+ document.body.innerHTML = domWithA11yIncompleteIssues;
+ process.env.SA11Y_CUSTOM_RULES = customRulesFilePath;
+ await expect(
+ automaticCheck({ cleanupAfterEach: true, enableIncompleteResults: true, consolidateResults: true })
+ ).rejects.toThrow('1 Accessibility');
+ delete process.env.SA11Y_CUSTOM_RULES;
+ });
+
  it('should take only custom rules if specified/testing for new rule', async () => {
  setup();
  document.body.innerHTML = domWithA11yCustomIssues;