feat: Add ActTestRunner class

readme.md

@@ -18,7 +18,90 @@ cd act-tools
 yarn
 ```
 
-## Usage
+## ActTestRunner class
+
+The ActTestRunner class helps accessibility tools to run ACT test cases, and turn the outcome into a valid EARL report. The following example shows how to run the test cases using Playwright.
+
+```js
+import { chromium } from 'playwright';
+import { ActTestRunner } from 'act-tools';
+import myA11yTool from './myA11yTool';
+
+(async () => {
+  const browser = await chromium.launch();
+  const page = await browser.newPage();
+  // Create a new instance of the tool runner
+  const actRunner = new ActTestRunner({
+    fileTypes: ['html'] // Only test HTML files
+    implementor: { // Set metadata about the tool
+      name: 'My Accessibility Tool',
+      versionNumber: '1.0.0'
+    }
+  });
+
+  // Run the tool against each HTML test case:
+  const actReport = await actRunner.run(async (testCase) => {
+    await page.goto(testCase.url); // Playwright load the page
+    const issues = await myA11yTool.a11yTest(page); // Run myA11yTool
+
+    // Convert myA11yTool's output to one the reporter understands
+    return issues.map(issue => ({
+      outcome: 'failed',
+      ruleId: issue.id,
+      wcag2: issue.wcagCriteria
+    }))
+  });
+
+  // Save the EARL report to disk
+  const earlText = JSON.stringify(actReport.getEarlReport(), null, 2);
+  writeFileSync('./earl-report.json', earlText, 'utf8');
+
+  // Console log a summary of the implementation
+  const { approvedRules, proposedRules } = report.getImplementationMapping();
+  console.table({ approvedRules, proposedRules });
+})();
+```
+
+### ActTestRunner#constructor
+
+The ActTestRunner constructor takes a single object as an argument, with the following options:
+
+- `implementor`: Object with the following optional properties:
+    - `name`: Name of the accessibility test tool
+    - `shortDesc`: Short description or title for the tool
+    - `versionNumber`: Version used in testing
+    - `vendorName`: Author of the tool
+- `rules`: Optional array of strings that limit which rules run
+- `fileTypes`: Optional array of strings that limit which file extensions are tested
+- `log`: Boolean, set to `false` to disable console logging
+- `gitVersion`: Branch name or commit from which to pull the `testcases.json` file. Defaults to `publication`, which is the latest version published on the WAI website.
+- `testCaseJsonUrl`: URL from which to load the testcases.json file. Setting this overrides `gitVersion`
+
+### run(testRunner)
+
+The function passed into `actTestRunner.run()` is called the testRunner. The test runner is passed a single testCase object with the following properties and method:
+
+- `fetchSource()`; Async method that returns the source code of the test case as a string
+- `ruleId`: 6-character ID given to the ACT rule
+- `ruleName`: Title given to the ACT rule
+- `testcaseId`: 48 character hash of the test case code
+- `testcaseTitle`: Name of the test case such as "Passed example 1"
+- `url`: Absolute path to the test case
+- `relativePath`: Unique path for the test case
+- `expected`: `passed`, `failed`, or `inapplicable`
+- `rulePage`: URL to the ACT rule page;
+- `ruleAccessibilityRequirements`: null, or Record of accessibility requirements
+
+The testRunner must return an array of objects with the following properties:
+
+- `procedureId` (required): Unique identifier or name for the procedure / check / rule the tool tested.
+- `outcome` (required): `passed`, `failed`, `inapplicable`, or `cantTell`.
+- `wcag2`: Array of WCAG 2 success criteria numbers that fail if the procedure fails
+- `requirements`: Array of other accessibility requirements that fail if the procedure fails, such as ARIA Practices and WCAG Techniques
+
+Tools can return more than one result for a procedure. For example if the test case as two links without a name, the tool may return two `failed` outcomes. Many accessibility tools report warning in addition to violations. Warnings should be reported as `cantTell`. 
+
+## CLI Usage
 
 ### Rule Transformer
 

src/EarlReport/EarlAssertion.ts

@@ -7,30 +7,31 @@ export type Outcome =
 
 export interface AssertionSpec {
   outcome: Outcome;
-  ruleId: string;
+  procedureId: string;
   wcag2?: string[];
   requirements?: string[];
 }
 
 export class EarlAssertion {
   outcome: Outcome;
-  ruleId: string;
+  procedureId: string;
   wcag2: string[];
   requirements: string[];
 
   constructor({
     outcome,
-    ruleId,
+    procedureId,
     wcag2 = [],
     requirements = [],
   }: AssertionSpec) {
     this.outcome = outcome;
-    this.ruleId = ruleId;
+    this.procedureId = procedureId;
     this.wcag2 = wcag2;
     this.requirements = requirements;
   }
 
-  get isPartOf(): void | object {
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+  get isPartOf() {
     if (!this.requirements.length && !this.wcag2.length) {
       return;
     }
@@ -43,7 +44,8 @@ export class EarlAssertion {
     ];
   }
 
-  toJSON(): object {
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+  toJSON() {
     return {
       "@type": "Assertion",
       result: {
@@ -52,7 +54,7 @@ export class EarlAssertion {
       },
       test: {
         "@type": "TestCase",
-        title: this.ruleId,
+        title: this.procedureId,
         isPartOf: this.isPartOf,
       },
       assertedBy: "_:assertor",

src/EarlReport/EarlAssertor.ts

@@ -1,28 +1,45 @@
 export interface AssertorSpec {
   name?: string;
   shortDesc?: string;
-  // versionNumber?: string
+  versionNumber?: string;
+  vendorName?: string;
   // versionDate?: string
 }
 
 export class EarlAssertor {
   name?: string;
   shortDesc?: string;
-  // versionNumber?: string
-  // versionDate?: string
+  versionNumber?: string;
+  versionDate?: string;
+  vendorName?: string;
 
-  constructor(assertor: AssertorSpec) {
+  constructor(assertor: AssertorSpec = {}) {
     this.name = assertor.name;
     this.shortDesc = assertor.shortDesc;
-    // this.versionNumber = assertor.versionNumber;
-    // this.versionDate = assertor.versionDate;
+    this.versionNumber = assertor.versionNumber;
+    this.vendorName = assertor.vendorName;
   }
 
-  toJSON(): object {
-    return {
+  toJSON(): Record<string, unknown> {
+    const json: Record<string, unknown> = {
       "@type": "Assertor",
       name: this.name,
       shortDesc: this.shortDesc,
     };
+
+    if (this.versionNumber) {
+      json.release = {
+        "@type": "Version",
+        revision: this.versionNumber,
+      };
+    }
+
+    if (this.vendorName) {
+      json.vendor = {
+        "@type": "Organization",
+        name: this.vendorName,
+      };
+    }
+    return json;
   }
 }

src/EarlReport/EarlReport.ts

@@ -6,7 +6,7 @@ export default class EarlReport {
   assertor: EarlAssertor;
   subjects: EarlSubject[] = [];
 
-  constructor(assertor: AssertorSpec) {
+  constructor(assertor: AssertorSpec = {}) {
     this.assertor = new EarlAssertor(assertor);
   }
 
@@ -16,7 +16,8 @@ export default class EarlReport {
     return subject;
   }
 
-  toJSON(): object {
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+  toJSON() {
     const report = {
       ...this.assertor.toJSON(),
       testRuns: this.subjects.map((subject) => subject.toJSON()),
@@ -26,3 +27,7 @@ export default class EarlReport {
     return report;
   }
 }
+
+export { AssertorSpec, EarlAssertor } from "./EarlAssertor";
+export { EarlSubject } from "./EarlSubject";
+export { EarlAssertion, AssertionSpec } from "./EarlAssertion";

src/EarlReport/EarlSubject.ts

@@ -7,12 +7,14 @@ export class EarlSubject {
     this.source = source;
   }
 
-  addAssertion(assertionSpec: AssertionSpec): void {
+  addAssertion(assertionSpec: AssertionSpec): EarlSubject {
     const assertion = new EarlAssertion(assertionSpec);
     this.assertions.push(assertion);
+    return this;
   }
 
-  toJSON(): object {
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+  toJSON() {
     return {
       "@type": ["TestSubject", "WebPage"],
       source: this.source,

src/EarlReport/context.ts

@@ -4,9 +4,12 @@ export const context = {
   WCAG2: "http://www.w3.org/TR/WCAG21/#",
   dct: "http://purl.org/dc/terms/",
   sch: "https://schema.org/",
+  doap: "http://usefulinc.com/ns/doap#",
   testRuns: "@graph",
-  name: "dct:name",
-  shortDesc: "dct:shortdesc",
+  name: "doap:name",
+  shortDesc: "doap:shortdesc",
+  release: "doap:release",
+  revision: "doap:revision",
   source: "dct:source",
   title: "dct:title",
   WebPage: "sch:WebPage",