Spike AXE A11Y testing in Cypress (#9111)

* Spike AXE A11Y testing in Cypress

* Fix NewRoomIntro breaking html/aria list rules

* Fix HeaderButtons breaking aria role semantics rules

* missing type

* Switch left panel from aside to nav and include space panel

* Give the page a main heading of the room name when viewing a room

* Use header landmark on RoomHeader

* Improve aria attributes on composer when autocomplete is closed

* Fix aria-owns on RoomHeader

* Give Spinner an aria role

* Give server picker help button an aria label

* Improve auth aria attributes and semantics

* Improve heading semantics in use case selection screen

* Fix autocomplete attribute to be valid

* Fix heading semantics on login page

* Improve Cypress axe testing

* Add axe tests

* Stop synapse after the timeline tests

* Await spinners to fade before percy snapshotting timeline tests

* Improve naming of plugin

* Update snapshots

* Fix accidental heading change

* Fix double synapse stoppage

* Fix Cypress timeline avatar assertions to be DPI agnostic

* Fix aria attributes on date separators

* delint

* Update snapshots

* Revert style change

* Skip redundant call

cypress/e2e/1-register/register.spec.ts

@@ -33,9 +33,12 @@ describe("Registration", () => {
     });
 
     it("registers an account and lands on the home screen", () => {
+        cy.injectAxe();
+
         cy.get(".mx_ServerPicker_change", { timeout: 15000 }).click();
         cy.get(".mx_ServerPickerDialog_continue").should("be.visible");
         cy.percySnapshot("Server Picker");
+        cy.checkA11y();
 
         cy.get(".mx_ServerPickerDialog_otherHomeserver").type(synapse.baseUrl);
         cy.get(".mx_ServerPickerDialog_continue").click();
@@ -46,6 +49,7 @@ describe("Registration", () => {
         // Hide the server text as it contains the randomly allocated Synapse port
         const percyCSS = ".mx_ServerPicker_server { visibility: hidden !important; }";
         cy.percySnapshot("Registration", { percyCSS });
+        cy.checkA11y();
 
         cy.get("#mx_RegistrationForm_username").type("alice");
         cy.get("#mx_RegistrationForm_password").type("totally a great password");
@@ -55,16 +59,21 @@ describe("Registration", () => {
 
         cy.get(".mx_RegistrationEmailPromptDialog").should("be.visible");
         cy.percySnapshot("Registration email prompt", { percyCSS });
+        cy.checkA11y();
         cy.get(".mx_RegistrationEmailPromptDialog button.mx_Dialog_primary").click();
 
         cy.stopMeasuring("create-account");
         cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy").should("be.visible");
         cy.percySnapshot("Registration terms prompt", { percyCSS });
+        cy.checkA11y();
 
         cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy input").click();
         cy.startMeasuring("from-submit-to-home");
         cy.get(".mx_InteractiveAuthEntryComponents_termsSubmit").click();
 
+        cy.get(".mx_UseCaseSelection_skip").should("exist");
+        cy.percySnapshot("Use-case selection screen");
+        cy.checkA11y();
         cy.get(".mx_UseCaseSelection_skip .mx_AccessibleButton").click();
 
         cy.url().should('contain', '/#/home');

cypress/e2e/14-timeline/timeline.spec.ts

@@ -20,7 +20,6 @@ import { MessageEvent } from "matrix-events-sdk";
 
 import type { ISendEventResponse } from "matrix-js-sdk/src/@types/requests";
 import type { EventType } from "matrix-js-sdk/src/@types/event";
-import type { MatrixClient } from "matrix-js-sdk/src/client";
 import { SynapseInstance } from "../../plugins/synapsedocker";
 import { SettingLevel } from "../../../src/settings/SettingLevel";
 import { Layout } from "../../../src/settings/enums/Layout";
@@ -46,10 +45,14 @@ const expectDisplayName = (e: JQuery<HTMLElement>, displayName: string): void =>
 };
 
 const expectAvatar = (e: JQuery<HTMLElement>, avatarUrl: string): void => {
-    cy.getClient().then((cli: MatrixClient) => {
+    cy.all([
+        cy.window({ log: false }),
+        cy.getClient(),
+    ]).then(([win, cli]) => {
+        const size = AVATAR_SIZE * win.devicePixelRatio;
         expect(e.find(".mx_BaseAvatar_image").attr("src")).to.equal(
             // eslint-disable-next-line no-restricted-properties
-            cli.mxcUrlToHttp(avatarUrl, AVATAR_SIZE, AVATAR_SIZE, AVATAR_RESIZE_METHOD),
+            cli.mxcUrlToHttp(avatarUrl, size, size, AVATAR_RESIZE_METHOD),
         );
     });
 };
@@ -75,15 +78,17 @@ describe("Timeline", () => {
         cy.startSynapse("default").then(data => {
             synapse = data;
             cy.initTestUser(synapse, OLD_NAME).then(() =>
-                cy.window({ log: false }).then(() => {
-                    cy.createRoom({ name: ROOM_NAME }).then(_room1Id => {
-                        roomId = _room1Id;
-                    });
+                cy.createRoom({ name: ROOM_NAME }).then(_room1Id => {
+                    roomId = _room1Id;
                 }),
             );
         });
     });
 
+    afterEach(() => {
+        cy.stopSynapse(synapse);
+    });
+
     describe("useOnlyCurrentProfiles", () => {
         beforeEach(() => {
             cy.uploadContent(OLD_AVATAR).then((url) => {
@@ -95,10 +100,6 @@ describe("Timeline", () => {
             });
         });
 
-        afterEach(() => {
-            cy.stopSynapse(synapse);
-        });
-
         it("should show historical profiles if disabled", () => {
             cy.setSettingValue("useOnlyCurrentProfiles", null, SettingLevel.ACCOUNT, false);
             sendEvent(roomId);
@@ -146,11 +147,16 @@ describe("Timeline", () => {
     });
 
     describe("message displaying", () => {
+        beforeEach(() => {
+            cy.injectAxe();
+        });
+
         it("should create and configure a room on IRC layout", () => {
             cy.visit("/#/room/" + roomId);
             cy.setSettingValue("layout", null, SettingLevel.DEVICE, Layout.IRC);
             cy.contains(".mx_RoomView_body .mx_GenericEventListSummary[data-layout=irc] " +
                 ".mx_GenericEventListSummary_summary", "created and configured the room.");
+            cy.get(".mx_Spinner").should("not.exist");
             cy.percySnapshot("Configured room on IRC layout");
         });
 
@@ -174,10 +180,12 @@ describe("Timeline", () => {
                 .should('have.css', "margin-inline-start", "104px")
                 .should('have.css', "inset-inline-start", "0px");
 
+            cy.get(".mx_Spinner").should("not.exist");
             // Exclude timestamp from snapshot
             const percyCSS = ".mx_RoomView_body .mx_EventTile_info .mx_MessageTimestamp "
                 + "{ visibility: hidden !important; }";
             cy.percySnapshot("Event line with inline start margin on IRC layout", { percyCSS });
+            cy.checkA11y();
         });
 
         it("should set inline start padding to a hidden event line", () => {

cypress/e2e/2-login/login.spec.ts

@@ -41,8 +41,11 @@ describe("Login", () => {
         });
 
         it("logs in with an existing account and lands on the home screen", () => {
+            cy.injectAxe();
+
             cy.get("#mx_LoginForm_username", { timeout: 15000 }).should("be.visible");
             cy.percySnapshot("Login");
+            cy.checkA11y();
 
             cy.get(".mx_ServerPicker_change").click();
             cy.get(".mx_ServerPickerDialog_otherHomeserver").type(synapse.baseUrl);

cypress/plugins/index.ts

@@ -22,6 +22,7 @@ import { performance } from "./performance";
 import { synapseDocker } from "./synapsedocker";
 import { webserver } from "./webserver";
 import { docker } from "./docker";
+import { log } from "./log";
 
 /**
  * @type {Cypress.PluginConfig}
@@ -31,4 +32,5 @@ export default function(on: PluginEvents, config: PluginConfigOptions) {
     performance(on, config);
     synapseDocker(on, config);
     webserver(on, config);
+    log(on, config);
 }

cypress/plugins/log.ts

@@ -0,0 +1,35 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/// <reference types="cypress" />
+
+import PluginEvents = Cypress.PluginEvents;
+import PluginConfigOptions = Cypress.PluginConfigOptions;
+
+export function log(on: PluginEvents, config: PluginConfigOptions) {
+    on("task", {
+        log(message: string) {
+            console.log(message);
+
+            return null;
+        },
+        table(message: string) {
+            console.table(message);
+
+            return null;
+        },
+    });
+}

cypress/support/axe.ts

@@ -0,0 +1,61 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/// <reference types="cypress" />
+
+import "cypress-axe";
+import * as axe from "axe-core";
+import { Options } from "cypress-axe";
+
+import Chainable = Cypress.Chainable;
+
+function terminalLog(violations: axe.Result[]): void {
+    cy.task(
+        'log',
+        `${violations.length} accessibility violation${
+            violations.length === 1 ? '' : 's'
+        } ${violations.length === 1 ? 'was' : 'were'} detected`,
+    );
+
+    // pluck specific keys to keep the table readable
+    const violationData = violations.map(({ id, impact, description, nodes }) => ({
+        id,
+        impact,
+        description,
+        nodes: nodes.length,
+    }));
+
+    cy.task('table', violationData);
+}
+
+Cypress.Commands.overwrite("checkA11y", (
+    originalFn: Chainable["checkA11y"],
+    context?: string | Node | axe.ContextObject | undefined,
+    options: Options = {},
+    violationCallback?: ((violations: axe.Result[]) => void) | undefined,
+    skipFailures?: boolean,
+): void => {
+    return originalFn(context, {
+        ...options,
+        rules: {
+            // Disable contrast checking for now as we have too many issues with it
+            'color-contrast': {
+                enabled: false,
+            },
+            ...options.rules,
+        },
+    }, violationCallback ?? terminalLog, skipFailures);
+});

cypress/support/e2e.ts

@@ -36,3 +36,4 @@ import "./iframes";
 import "./timeline";
 import "./network";
 import "./composer";
+import "./axe";

cypress/tsconfig.json

@@ -7,7 +7,11 @@
             "dom",
             "dom.iterable"
         ],
-        "types": ["cypress", "@percy/cypress"],
+        "types": [
+            "cypress",
+            "cypress-axe",
+            "@percy/cypress"
+        ],
         "resolveJsonModule": true,
         "esModuleInterop": true,
         "moduleResolution": "node",