Migrate remaining crypto tests from Cypress to Playwright (#12021)

* Fix bot MatrixClient being set up multiple times Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Migrate verification.spec.ts from Cypress to Playwright Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Migrate crypto.spec.ts from Cypress to Playwright Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * delint Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Add screenshot Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Record trace on-first-retry Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Don't start client when not needed Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Add bot log prefixing Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Turns out we need these Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix crypto tests in rust crypto Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --------- Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2023-12-12 08:55:29 +00:00 · 2023-12-12 08:55:29 +00:00 · 5104d53ddf
commit 5104d53ddf
parent 0f42418b5c
13 changed files with 1111 additions and 1259 deletions
--- a/playwright/e2e/crypto/crypto.spec.ts
+++ b/playwright/e2e/crypto/crypto.spec.ts
@ -0,0 +1,487 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import type { Page } from "@playwright/test";
+import { test, expect } from "../../element-web-test";
+import {
+    createSharedRoomWithUser,
+    doTwoWaySasVerification,
+    copyAndContinue,
+    enableKeyBackup,
+    logIntoElement,
+    logOutOfElement,
+    waitForVerificationRequest,
+} from "./utils";
+import { Bot } from "../../pages/bot";
+import { ElementAppPage } from "../../pages/ElementAppPage";
+import { Client } from "../../pages/client";
+
+const openRoomInfo = async (page: Page) => {
+    await page.getByRole("button", { name: "Room info" }).click();
+    return page.locator(".mx_RightPanel");
+};
+
+const checkDMRoom = async (page: Page) => {
+    const body = page.locator(".mx_RoomView_body");
+    await expect(body.getByText("Alice created this DM.")).toBeVisible();
+    await expect(body.getByText("Alice invited Bob")).toBeVisible({ timeout: 1000 });
+    await expect(body.locator(".mx_cryptoEvent").getByText("Encryption enabled")).toBeVisible();
+};
+
+const startDMWithBob = async (page: Page, bob: Bot) => {
+    await page.locator(".mx_RoomList").getByRole("button", { name: "Start chat" }).click();
+    await page.getByTestId("invite-dialog-input").fill(bob.credentials.userId);
+    await page.locator(".mx_InviteDialog_tile_nameStack_name").getByText("Bob").click();
+    await expect(
+        page.locator(".mx_InviteDialog_userTile_pill .mx_InviteDialog_userTile_name").getByText("Bob"),
+    ).toBeVisible();
+    await page.getByRole("button", { name: "Go" }).click();
+};
+
+const testMessages = async (page: Page, bob: Bot, bobRoomId: string) => {
+    // check the invite message
+    await expect(
+        page.locator(".mx_EventTile", { hasText: "Hey!" }).locator(".mx_EventTile_e2eIcon_warning"),
+    ).not.toBeVisible();
+
+    // Bob sends a response
+    await bob.sendMessage(bobRoomId, "Hoo!");
+    await expect(
+        page.locator(".mx_EventTile", { hasText: "Hoo!" }).locator(".mx_EventTile_e2eIcon_warning"),
+    ).not.toBeVisible();
+};
+
+const bobJoin = async (page: Page, bob: Bot) => {
+    await bob.evaluate(async (cli) => {
+        const bobRooms = cli.getRooms();
+        if (!bobRooms.length) {
+            await new Promise<void>((resolve) => {
+                const onMembership = (_event) => {
+                    cli.off(window.matrixcs.RoomMemberEvent.Membership, onMembership);
+                    resolve();
+                };
+                cli.on(window.matrixcs.RoomMemberEvent.Membership, onMembership);
+            });
+        }
+    });
+    const roomId = await bob.joinRoomByName("Alice");
+
+    await expect(page.getByText("Bob joined the room")).toBeVisible();
+    return roomId;
+};
+
+/** configure the given MatrixClient to auto-accept any invites */
+async function autoJoin(client: Client) {
+    await client.evaluate((cli) => {
+        cli.on(window.matrixcs.RoomMemberEvent.Membership, (event, member) => {
+            if (member.membership === "invite" && member.userId === cli.getUserId()) {
+                cli.joinRoom(member.roomId);
+            }
+        });
+    });
+}
+
+const verify = async (page: Page, bob: Bot) => {
+    const bobsVerificationRequestPromise = waitForVerificationRequest(bob);
+
+    const roomInfo = await openRoomInfo(page);
+    await roomInfo.getByRole("menuitem", { name: "People" }).click();
+    await roomInfo.getByText("Bob").click();
+    await roomInfo.getByRole("button", { name: "Verify" }).click();
+    await roomInfo.getByRole("button", { name: "Start Verification" }).click();
+
+    // this requires creating a DM, so can take a while. Give it a longer timeout.
+    await roomInfo.getByRole("button", { name: "Verify by emoji" }).click({ timeout: 30000 });
+
+    const request = await bobsVerificationRequestPromise;
+    // the bot user races with the Element user to hit the "verify by emoji" button
+    const verifier = await request.evaluateHandle((request) => request.startVerification("m.sas.v1"));
+    await doTwoWaySasVerification(page, verifier);
+    await roomInfo.getByRole("button", { name: "They match" }).click();
+    await expect(roomInfo.getByText("You've successfully verified Bob!")).toBeVisible();
+    await roomInfo.getByRole("button", { name: "Got it" }).click();
+};
+
+test.describe("Cryptography", function () {
+    test.use({
+        displayName: "Alice",
+        botCreateOpts: {
+            displayName: "Bob",
+            autoAcceptInvites: false,
+            // XXX: We use a custom prefix here to coerce the Rust Crypto SDK to prefer `@user` in race resolution
+            // by using a prefix that is lexically after `@user` in the alphabet.
+            userIdPrefix: "zzz_",
+        },
+    });
+
+    for (const isDeviceVerified of [true, false]) {
+        test.describe(`setting up secure key backup should work isDeviceVerified=${isDeviceVerified}`, () => {
+            /**
+             * Verify that the `m.cross_signing.${keyType}` key is available on the account data on the server
+             * @param keyType
+             */
+            async function verifyKey(app: ElementAppPage, keyType: string) {
+                const accountData: { encrypted: Record<string, Record<string, string>> } = await app.client.evaluate(
+                    (cli, keyType) => cli.getAccountDataFromServer(`m.cross_signing.${keyType}`),
+                    keyType,
+                );
+                expect(accountData.encrypted).toBeDefined();
+                const keys = Object.keys(accountData.encrypted);
+                const key = accountData.encrypted[keys[0]];
+                expect(key.ciphertext).toBeDefined();
+                expect(key.iv).toBeDefined();
+                expect(key.mac).toBeDefined();
+            }
+
+            test("by recovery code", async ({ page, app, user: aliceCredentials }) => {
+                // Verified the device
+                if (isDeviceVerified) {
+                    await app.client.bootstrapCrossSigning(aliceCredentials);
+                }
+
+                await app.settings.openUserSettings("Security & Privacy");
+                await page.getByRole("button", { name: "Set up Secure Backup" }).click();
+
+                const dialog = page.locator(".mx_Dialog");
+                // Recovery key is selected by default
+                await dialog.getByRole("button", { name: "Continue" }).click();
+                await copyAndContinue(page);
+
+                // When the device is verified, the `Setting up keys` step is skipped
+                if (!isDeviceVerified) {
+                    const uiaDialogTitle = page.locator(".mx_InteractiveAuthDialog .mx_Dialog_title");
+                    await expect(uiaDialogTitle.getByText("Setting up keys")).toBeVisible();
+                    await expect(uiaDialogTitle.getByText("Setting up keys")).not.toBeVisible();
+                }
+
+                await expect(dialog.getByText("Secure Backup successful")).toBeVisible();
+                await dialog.getByRole("button", { name: "Done" }).click();
+                await expect(dialog.getByText("Secure Backup successful")).not.toBeVisible();
+
+                // Verify that the SSSS keys are in the account data stored in the server
+                await verifyKey(app, "master");
+                await verifyKey(app, "self_signing");
+                await verifyKey(app, "user_signing");
+            });
+
+            test("by passphrase", async ({ page, app, user: aliceCredentials }) => {
+                // Verified the device
+                if (isDeviceVerified) {
+                    await app.client.bootstrapCrossSigning(aliceCredentials);
+                }
+
+                await app.settings.openUserSettings("Security & Privacy");
+                await page.getByRole("button", { name: "Set up Secure Backup" }).click();
+
+                const dialog = page.locator(".mx_Dialog");
+                // Select passphrase option
+                await dialog.getByText("Enter a Security Phrase").click();
+                await dialog.getByRole("button", { name: "Continue" }).click();
+
+                // Fill passphrase input
+                await dialog.locator("input").fill("new passphrase for setting up a secure key backup");
+                await dialog.locator(".mx_Dialog_primary:not([disabled])", { hasText: "Continue" }).click();
+                // Confirm passphrase
+                await dialog.locator("input").fill("new passphrase for setting up a secure key backup");
+                await dialog.locator(".mx_Dialog_primary:not([disabled])", { hasText: "Continue" }).click();
+
+                await copyAndContinue(page);
+
+                await expect(dialog.getByText("Secure Backup successful")).toBeVisible();
+                await dialog.getByRole("button", { name: "Done" }).click();
+                await expect(dialog.getByText("Secure Backup successful")).not.toBeVisible();
+
+                // Verify that the SSSS keys are in the account data stored in the server
+                await verifyKey(app, "master");
+                await verifyKey(app, "self_signing");
+                await verifyKey(app, "user_signing");
+            });
+        });
+    }
+
+    test("creating a DM should work, being e2e-encrypted / user verification", async ({
+        page,
+        app,
+        bot: bob,
+        user: aliceCredentials,
+    }) => {
+        await app.client.bootstrapCrossSigning(aliceCredentials);
+        await startDMWithBob(page, bob);
+        // send first message
+        await page.getByRole("textbox", { name: "Send a message…" }).fill("Hey!");
+        await page.getByRole("textbox", { name: "Send a message…" }).press("Enter");
+        await checkDMRoom(page);
+        const bobRoomId = await bobJoin(page, bob);
+        await testMessages(page, bob, bobRoomId);
+        await verify(page, bob);
+
+        // Assert that verified icon is rendered
+        await page.getByRole("button", { name: "Room members" }).click();
+        await page.getByRole("button", { name: "Room information" }).click();
+        await expect(page.locator('.mx_RoomSummaryCard_badges [data-kind="success"]')).toContainText("Encrypted");
+
+        // Take a snapshot of RoomSummaryCard with a verified E2EE icon
+        await expect(page.locator(".mx_RightPanel")).toMatchScreenshot("RoomSummaryCard-with-verified-e2ee.png");
+    });
+
+    test("should allow verification when there is no existing DM", async ({
+        page,
+        app,
+        bot: bob,
+        user: aliceCredentials,
+    }) => {
+        await app.client.bootstrapCrossSigning(aliceCredentials);
+        await autoJoin(bob);
+
+        // we need to have a room with the other user present, so we can open the verification panel
+        await createSharedRoomWithUser(app, bob.credentials.userId);
+        await verify(page, bob);
+    });
+
+    test.describe("event shields", () => {
+        let testRoomId: string;
+
+        test.beforeEach(async ({ page, bot: bob, user: aliceCredentials, app }) => {
+            await app.client.bootstrapCrossSigning(aliceCredentials);
+            await autoJoin(bob);
+
+            // create an encrypted room
+            testRoomId = await createSharedRoomWithUser(app, bob.credentials.userId, {
+                name: "TestRoom",
+                initial_state: [
+                    {
+                        type: "m.room.encryption",
+                        state_key: "",
+                        content: {
+                            algorithm: "m.megolm.v1.aes-sha2",
+                        },
+                    },
+                ],
+            });
+        });
+
+        test("should show the correct shield on e2e events", async ({
+            page,
+            app,
+            bot: bob,
+            homeserver,
+            cryptoBackend,
+        }) => {
+            // Bob has a second, not cross-signed, device
+            const bobSecondDevice = new Bot(page, homeserver, {
+                bootstrapSecretStorage: false,
+                bootstrapCrossSigning: false,
+            });
+            bobSecondDevice.setCredentials(
+                await homeserver.loginUser(bob.credentials.userId, bob.credentials.password),
+            );
+            await bobSecondDevice.prepareClient();
+
+            await bob.sendEvent(testRoomId, null, "m.room.encrypted", {
+                algorithm: "m.megolm.v1.aes-sha2",
+                ciphertext: "the bird is in the hand",
+            });
+
+            const last = page.locator(".mx_EventTile_last");
+            await expect(last).toContainText("Unable to decrypt message");
+            const lastE2eIcon = last.locator(".mx_EventTile_e2eIcon");
+            await expect(lastE2eIcon).toHaveClass(/mx_EventTile_e2eIcon_decryption_failure/);
+            await expect(lastE2eIcon).toHaveAttribute("aria-label", "This message could not be decrypted");
+
+            /* Should show a red padlock for an unencrypted message in an e2e room */
+            await bob.evaluate(
+                (cli, testRoomId) =>
+                    cli.http.authedRequest(
+                        window.matrixcs.Method.Put,
+                        `/rooms/${encodeURIComponent(testRoomId)}/send/m.room.message/test_txn_1`,
+                        undefined,
+                        {
+                            msgtype: "m.text",
+                            body: "test unencrypted",
+                        },
+                    ),
+                testRoomId,
+            );
+
+            await expect(last).toContainText("test unencrypted");
+            await expect(lastE2eIcon).toHaveClass(/mx_EventTile_e2eIcon_warning/);
+            await expect(lastE2eIcon).toHaveAttribute("aria-label", "Not encrypted");
+
+            /* Should show no padlock for an unverified user */
+            // bob sends a valid event
+            await bob.sendMessage(testRoomId, "test encrypted 1");
+
+            // the message should appear, decrypted, with no warning, but also no "verified"
+            const lastTile = page.locator(".mx_EventTile_last");
+            const lastTileE2eIcon = lastTile.locator(".mx_EventTile_e2eIcon");
+            await expect(lastTile).toContainText("test encrypted 1");
+            // no e2e icon
+            await expect(lastTileE2eIcon).not.toBeVisible();
+
+            /* Now verify Bob */
+            await verify(page, bob);
+
+            /* Existing message should be updated when user is verified. */
+            await expect(last).toContainText("test encrypted 1");
+            // still no e2e icon
+            await expect(last.locator(".mx_EventTile_e2eIcon")).not.toBeVisible();
+
+            /* should show no padlock, and be verified, for a message from a verified device */
+            await bob.sendMessage(testRoomId, "test encrypted 2");
+
+            await expect(lastTile).toContainText("test encrypted 2");
+            // no e2e icon
+            await expect(lastTileE2eIcon).not.toBeVisible();
+
+            /* should show red padlock for a message from an unverified device */
+            await bobSecondDevice.sendMessage(testRoomId, "test encrypted from unverified");
+            await expect(lastTile).toContainText("test encrypted from unverified");
+            await expect(lastTileE2eIcon).toHaveClass(/mx_EventTile_e2eIcon_warning/);
+            await expect(lastTileE2eIcon).toHaveAttribute(
+                "aria-label",
+                "Encrypted by a device not verified by its owner.",
+            );
+
+            /* Should show a grey padlock for a message from an unknown device */
+            // bob deletes his second device
+            await bobSecondDevice.evaluate((cli) => cli.logout(true));
+
+            // wait for the logout to propagate. Workaround for https://github.com/vector-im/element-web/issues/26263 by repeatedly closing and reopening Bob's user info.
+            async function awaitOneDevice(iterations = 1) {
+                const rightPanel = page.locator(".mx_RightPanel");
+                await rightPanel.getByRole("button", { name: "Room members" }).click();
+                await rightPanel.getByText("Bob").click();
+                const sessionCountText = await rightPanel
+                    .locator(".mx_UserInfo_devices")
+                    .getByText(" session", { exact: false })
+                    .textContent();
+                // cf https://github.com/vector-im/element-web/issues/26279: Element-R uses the wrong text here
+                if (sessionCountText != "1 session" && sessionCountText != "1 verified session") {
+                    if (iterations >= 10) {
+                        throw new Error(`Bob still has ${sessionCountText} after 10 iterations`);
+                    }
+                    await awaitOneDevice(iterations + 1);
+                }
+            }
+
+            await awaitOneDevice();
+
+            // close and reopen the room, to get the shield to update.
+            await app.viewRoomByName("Bob");
+            await app.viewRoomByName("TestRoom");
+
+            // some debate over whether this should have a red or a grey shield. Legacy crypto shows a grey shield,
+            // Rust crypto a red one.
+            await expect(last).toContainText("test encrypted from unverified");
+            if (cryptoBackend === "rust") {
+                await expect(lastE2eIcon).toHaveClass(/mx_EventTile_e2eIcon_warning/);
+            } else {
+                await expect(lastE2eIcon).toHaveClass(/mx_EventTile_e2eIcon_normal/);
+            }
+            await expect(lastE2eIcon).toHaveAttribute("aria-label", "Encrypted by an unknown or deleted device.");
+        });
+
+        // XXX: Failed since migration to Playwright
+        test.skip("Should show a grey padlock for a key restored from backup", async ({
+            page,
+            app,
+            bot: bob,
+            homeserver,
+            user: aliceCredentials,
+        }) => {
+            const securityKey = await enableKeyBackup(app);
+
+            // bob sends a valid event
+            await bob.sendMessage(testRoomId, "test encrypted 1");
+
+            const lastTile = page.locator(".mx_EventTile_last");
+            const lastTileE2eIcon = lastTile.locator(".mx_EventTile_e2eIcon");
+            await expect(lastTile).toContainText("test encrypted 1");
+            // no e2e icon
+            await expect(lastTileE2eIcon).not.toBeVisible();
+
+            // It can take up to 10 seconds for the key to be backed up. We don't really have much option other than
+            // to wait :/
+            await page.waitForTimeout(10000);
+
+            /* log out, and back in */
+            await logOutOfElement(page);
+            await logIntoElement(page, homeserver, aliceCredentials, securityKey);
+
+            /* go back to the test room and find Bob's message again */
+            await app.viewRoomById(testRoomId);
+            await expect(lastTile).toContainText("test encrypted 1");
+            await expect(lastTileE2eIcon).toHaveClass(/mx_EventTile_e2eIcon_warning/);
+            await expect(lastTileE2eIcon).toHaveAttribute("aria-label", "Encrypted by an unknown or deleted device.");
+        });
+
+        test("should show the correct shield on edited e2e events", async ({ page, app, bot: bob, homeserver }) => {
+            // bob has a second, not cross-signed, device
+            const bobSecondDevice = new Bot(page, homeserver, {
+                bootstrapSecretStorage: false,
+                bootstrapCrossSigning: false,
+            });
+            bobSecondDevice.setCredentials(
+                await homeserver.loginUser(bob.credentials.userId, bob.credentials.password),
+            );
+            await bobSecondDevice.prepareClient();
+
+            // verify Bob
+            await verify(page, bob);
+
+            // bob sends a valid event
+            const testEvent = await bob.sendMessage(testRoomId, "Hoo!");
+
+            // the message should appear, decrypted, with no warning
+            await expect(
+                page.locator(".mx_EventTile", { hasText: "Hoo!" }).locator(".mx_EventTile_e2eIcon_warning"),
+            ).not.toBeVisible();
+
+            // bob sends an edit to the first message with his unverified device
+            await bobSecondDevice.sendMessage(testRoomId, {
+                "m.new_content": {
+                    msgtype: "m.text",
+                    body: "Haa!",
+                },
+                "m.relates_to": {
+                    rel_type: "m.replace",
+                    event_id: testEvent.event_id,
+                },
+            });
+
+            // the edit should have a warning
+            await expect(
+                page.locator(".mx_EventTile", { hasText: "Haa!" }).locator(".mx_EventTile_e2eIcon_warning"),
+            ).toBeVisible();
+
+            // a second edit from the verified device should be ok
+            await bob.sendMessage(testRoomId, {
+                "m.new_content": {
+                    msgtype: "m.text",
+                    body: "Hee!",
+                },
+                "m.relates_to": {
+                    rel_type: "m.replace",
+                    event_id: testEvent.event_id,
+                },
+            });
+
+            await expect(
+                page.locator(".mx_EventTile", { hasText: "Hee!" }).locator(".mx_EventTile_e2eIcon_warning"),
+            ).not.toBeVisible();
+        });
+    });
+});
--- a/playwright/e2e/crypto/utils.ts
+++ b/playwright/e2e/crypto/utils.ts
@ -14,9 +14,102 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-import { type Page, expect } from "@playwright/test";
+import { type Page, expect, JSHandle } from "@playwright/test";

+import type { CryptoEvent, ICreateRoomOpts, MatrixClient } from "matrix-js-sdk/src/matrix";
+import type {
+    VerificationRequest,
+    Verifier,
+    EmojiMapping,
+    VerifierEvent,
+} from "matrix-js-sdk/src/crypto-api/verification";
+import type { ISasEvent } from "matrix-js-sdk/src/crypto/verification/SAS";
 import { Credentials, HomeserverInstance } from "../../plugins/homeserver";
+import { Client } from "../../pages/client";
+import { ElementAppPage } from "../../pages/ElementAppPage";
+
+/**
+ * wait for the given client to receive an incoming verification request, and automatically accept it
+ *
+ * @param client - matrix client handle we expect to receive a request
+ */
+export async function waitForVerificationRequest(client: Client): Promise<JSHandle<VerificationRequest>> {
+    return client.evaluateHandle((cli) => {
+        return new Promise<VerificationRequest>((resolve) => {
+            console.log("~~");
+            const onVerificationRequestEvent = async (request: VerificationRequest) => {
+                console.log("@@", request);
+                await request.accept();
+                resolve(request);
+            };
+            cli.once(
+                "crypto.verificationRequestReceived" as CryptoEvent.VerificationRequestReceived,
+                onVerificationRequestEvent,
+            );
+        });
+    });
+}
+
+/**
+ * Automatically handle a SAS verification
+ *
+ * Given a verifier which has already been started, wait for the emojis to be received, blindly confirm they
+ * match, and return them
+ *
+ * @param verifier - verifier
+ * @returns A promise that resolves, with the emoji list, once we confirm the emojis
+ */
+export function handleSasVerification(verifier: JSHandle<Verifier>): Promise<EmojiMapping[]> {
+    return verifier.evaluate((verifier) => {
+        const event = verifier.getShowSasCallbacks();
+        if (event) return event.sas.emoji;
+
+        return new Promise<EmojiMapping[]>((resolve) => {
+            const onShowSas = (event: ISasEvent) => {
+                verifier.off("show_sas" as VerifierEvent, onShowSas);
+                event.confirm();
+                resolve(event.sas.emoji);
+            };
+
+            verifier.on("show_sas" as VerifierEvent, onShowSas);
+        });
+    });
+}
+
+/**
+ * Check that the user has published cross-signing keys, and that the user's device has been cross-signed.
+ */
+export async function checkDeviceIsCrossSigned(app: ElementAppPage): Promise<void> {
+    const { userId, deviceId, keys } = await app.client.evaluate(async (cli: MatrixClient) => {
+        const deviceId = cli.getDeviceId();
+        const userId = cli.getUserId();
+        const keys = await cli.downloadKeysForUsers([userId]);
+
+        return { userId, deviceId, keys };
+    });
+
+    // there should be three cross-signing keys
+    expect(keys.master_keys[userId]).toHaveProperty("keys");
+    expect(keys.self_signing_keys[userId]).toHaveProperty("keys");
+    expect(keys.user_signing_keys[userId]).toHaveProperty("keys");
+
+    // and the device should be signed by the self-signing key
+    const selfSigningKeyId = Object.keys(keys.self_signing_keys[userId].keys)[0];
+
+    expect(keys.device_keys[userId][deviceId]).toBeDefined();
+
+    const myDeviceSignatures = keys.device_keys[userId][deviceId].signatures[userId];
+    expect(myDeviceSignatures[selfSigningKeyId]).toBeDefined();
+}
+
+/**
+ * Check that the current device is connected to the key backup.
+ */
+export async function checkDeviceIsConnectedKeyBackup(page: Page) {
+    await page.getByRole("button", { name: "User menu" }).click();
+    await page.locator(".mx_UserMenu_contextMenu").getByRole("menuitem", { name: "Security & Privacy" }).click();
+    await expect(page.locator(".mx_Dialog").getByRole("button", { name: "Restore from Backup" })).toBeVisible();
+}

 /**
 * Fill in the login form in element with the given creds.
@ -52,3 +145,95 @@ export async function logIntoElement(
        await page.getByRole("button", { name: "Done" }).click();
    }
 }
+
+export async function logOutOfElement(page: Page) {
+    await page.getByRole("button", { name: "User menu" }).click();
+    await page.locator(".mx_UserMenu_contextMenu").getByRole("menuitem", { name: "Sign out" }).click();
+    await page.locator(".mx_Dialog .mx_QuestionDialog").getByRole("button", { name: "Sign out" }).click();
+
+    // Wait for the login page to load
+    await page.getByRole("heading", { name: "Sign in" }).click();
+}
+
+/**
+ * Given a SAS verifier for a bot client:
+ *   - wait for the bot to receive the emojis
+ *   - check that the bot sees the same emoji as the application
+ *
+ * @param verifier - a verifier in a bot client
+ */
+export async function doTwoWaySasVerification(page: Page, verifier: JSHandle<Verifier>): Promise<void> {
+    // on the bot side, wait for the emojis, confirm they match, and return them
+    const emojis = await handleSasVerification(verifier);
+
+    const emojiBlocks = page.locator(".mx_VerificationShowSas_emojiSas_block");
+    await expect(emojiBlocks).toHaveCount(emojis.length);
+
+    // then, check that our application shows an emoji panel with the same emojis.
+    for (let i = 0; i < emojis.length; i++) {
+        const emoji = emojis[i];
+        const emojiBlock = emojiBlocks.nth(i);
+        const textContent = await emojiBlock.textContent();
+        // VerificationShowSas munges the case of the emoji descriptions returned by the js-sdk before
+        // displaying them. Once we drop support for legacy crypto, that code can go away, and so can the
+        // case-munging here.
+        expect(textContent.toLowerCase()).toEqual(emoji[0] + emoji[1].toLowerCase());
+    }
+}
+
+/**
+ * Open the security settings and enable secure key backup.
+ *
+ * Assumes that the current device has been cross-signed (which means that we skip a step where we set it up).
+ *
+ * Returns the security key
+ */
+export async function enableKeyBackup(app: ElementAppPage): Promise<string> {
+    await app.settings.openUserSettings("Security & Privacy");
+    await app.page.getByRole("button", { name: "Set up Secure Backup" }).click();
+    const dialog = app.page.locator(".mx_Dialog");
+    // Recovery key is selected by default
+    await dialog.getByRole("button", { name: "Continue" }).click({ timeout: 60000 });
+
+    // copy the text ourselves
+    const securityKey = await dialog.locator(".mx_CreateSecretStorageDialog_recoveryKey code").textContent();
+    await copyAndContinue(app.page);
+
+    await expect(dialog.getByText("Secure Backup successful")).toBeVisible();
+    await dialog.getByRole("button", { name: "Done" }).click();
+    await expect(dialog.getByText("Secure Backup successful")).not.toBeVisible();
+
+    return securityKey;
+}
+
+/**
+ * Click on copy and continue buttons to dismiss the security key dialog
+ */
+export async function copyAndContinue(page: Page) {
+    await page.getByRole("button", { name: "Copy" }).click();
+    await page.getByRole("button", { name: "Continue" }).click();
+}
+
+/**
+ * Create a shared, unencrypted room with the given user, and wait for them to join
+ *
+ * @param other - UserID of the other user
+ * @param opts - other options for the createRoom call
+ *
+ * @returns a promise which resolves to the room ID
+ */
+export async function createSharedRoomWithUser(
+    app: ElementAppPage,
+    other: string,
+    opts: Omit<ICreateRoomOpts, "invite"> = { name: "TestRoom" },
+): Promise<string> {
+    const roomId = await app.client.createRoom({ ...opts, invite: [other] });
+
+    await app.viewRoomById(roomId);
+
+    // wait for the other user to join the room, otherwise our attempt to open his user details may race
+    // with his join.
+    await expect(app.page.getByText(" joined the room", { exact: false })).toBeVisible();
+
+    return roomId;
+}
--- a/playwright/e2e/crypto/verification.spec.ts
+++ b/playwright/e2e/crypto/verification.spec.ts
@ -0,0 +1,348 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import jsQR from "jsqr";
+
+import type { JSHandle, Locator, Page } from "@playwright/test";
+import type { Preset, Visibility } from "matrix-js-sdk/src/matrix";
+import type { VerificationRequest, Verifier } from "matrix-js-sdk/src/crypto-api";
+import { test, expect } from "../../element-web-test";
+import {
+    checkDeviceIsConnectedKeyBackup,
+    checkDeviceIsCrossSigned,
+    doTwoWaySasVerification,
+    logIntoElement,
+    waitForVerificationRequest,
+} from "./utils";
+import { Client } from "../../pages/client";
+import { Bot } from "../../pages/bot";
+
+test.describe("Device verification", () => {
+    let aliceBotClient: Bot;
+
+    test.beforeEach(async ({ page, homeserver, credentials }) => {
+        // Visit the login page of the app, to load the matrix sdk
+        await page.goto("/#/login");
+
+        await page.pause();
+
+        // wait for the page to load
+        await page.waitForSelector(".mx_AuthPage", { timeout: 30000 });
+
+        // Create a new device for alice
+        aliceBotClient = new Bot(page, homeserver, {
+            rustCrypto: true,
+            bootstrapCrossSigning: true,
+            bootstrapSecretStorage: true,
+        });
+        aliceBotClient.setCredentials(credentials);
+        await aliceBotClient.prepareClient();
+
+        await page.waitForTimeout(20000);
+    });
+
+    // Click the "Verify with another device" button, and have the bot client auto-accept it.
+    async function initiateAliceVerificationRequest(page: Page): Promise<JSHandle<VerificationRequest>> {
+        // alice bot waits for verification request
+        const promiseVerificationRequest = waitForVerificationRequest(aliceBotClient);
+
+        // Click on "Verify with another device"
+        await page.locator(".mx_AuthPage").getByRole("button", { name: "Verify with another device" }).click();
+
+        // alice bot responds yes to verification request from alice
+        return promiseVerificationRequest;
+    }
+
+    test("Verify device with SAS during login", async ({ page, app, credentials, homeserver }) => {
+        await logIntoElement(page, homeserver, credentials);
+
+        // Launch the verification request between alice and the bot
+        const verificationRequest = await initiateAliceVerificationRequest(page);
+
+        // Handle emoji SAS verification
+        const infoDialog = page.locator(".mx_InfoDialog");
+        // the bot chooses to do an emoji verification
+        const verifier = await verificationRequest.evaluateHandle((request) => request.startVerification("m.sas.v1"));
+
+        // Handle emoji request and check that emojis are matching
+        await doTwoWaySasVerification(page, verifier);
+
+        await infoDialog.getByRole("button", { name: "They match" }).click();
+        await infoDialog.getByRole("button", { name: "Got it" }).click();
+
+        // Check that our device is now cross-signed
+        await checkDeviceIsCrossSigned(app);
+
+        // Check that the current device is connected to key backup
+        await checkDeviceIsConnectedKeyBackup(page);
+    });
+
+    test("Verify device with QR code during login", async ({ page, app, credentials, homeserver }) => {
+        // A mode 0x02 verification: "self-verifying in which the current device does not yet trust the master key"
+        await logIntoElement(page, homeserver, credentials);
+
+        // Launch the verification request between alice and the bot
+        const verificationRequest = await initiateAliceVerificationRequest(page);
+
+        const infoDialog = page.locator(".mx_InfoDialog");
+        // feed the QR code into the verification request.
+        const qrData = await readQrCode(infoDialog);
+        const verifier = await verificationRequest.evaluateHandle(
+            (request, qrData) => request.scanQRCode(new Uint8Array(qrData)),
+            [...qrData],
+        );
+
+        // Confirm that the bot user scanned successfully
+        await expect(infoDialog.getByText("Almost there! Is your other device showing the same shield?")).toBeVisible();
+        await infoDialog.getByRole("button", { name: "Yes" }).click();
+        await infoDialog.getByRole("button", { name: "Got it" }).click();
+
+        // wait for the bot to see we have finished
+        await verifier.evaluate((verifier) => verifier.verify());
+
+        // the bot uploads the signatures asynchronously, so wait for that to happen
+        await page.waitForTimeout(1000);
+
+        // our device should trust the bot device
+        await app.client.evaluate(async (cli, aliceBotCredentials) => {
+            const deviceStatus = await cli
+                .getCrypto()!
+                .getDeviceVerificationStatus(aliceBotCredentials.userId, aliceBotCredentials.deviceId);
+            if (!deviceStatus.isVerified()) {
+                throw new Error("Bot device was not verified after QR code verification");
+            }
+        }, aliceBotClient.credentials);
+
+        // Check that our device is now cross-signed
+        await checkDeviceIsCrossSigned(app);
+
+        // Check that the current device is connected to key backup
+        await checkDeviceIsConnectedKeyBackup(page);
+    });
+
+    test("Verify device with Security Phrase during login", async ({ page, app, credentials, homeserver }) => {
+        await logIntoElement(page, homeserver, credentials);
+
+        // Select the security phrase
+        await page.locator(".mx_AuthPage").getByRole("button", { name: "Verify with Security Key or Phrase" }).click();
+
+        // Fill the passphrase
+        const dialog = page.locator(".mx_Dialog");
+        await dialog.locator("input").fill("new passphrase");
+        await dialog.locator(".mx_Dialog_primary:not([disabled])", { hasText: "Continue" }).click();
+
+        await page.locator(".mx_AuthPage").getByRole("button", { name: "Done" }).click();
+
+        // Check that our device is now cross-signed
+        await checkDeviceIsCrossSigned(app);
+
+        // Check that the current device is connected to key backup
+        await checkDeviceIsConnectedKeyBackup(page);
+    });
+
+    test("Verify device with Security Key during login", async ({ page, app, credentials, homeserver }) => {
+        await logIntoElement(page, homeserver, credentials);
+
+        // Select the security phrase
+        await page.locator(".mx_AuthPage").getByRole("button", { name: "Verify with Security Key or Phrase" }).click();
+
+        // Fill the security key
+        const dialog = page.locator(".mx_Dialog");
+        await dialog.getByRole("button", { name: "use your Security Key" }).click();
+        const aliceRecoveryKey = await aliceBotClient.getRecoveryKey();
+        await dialog.locator("#mx_securityKey").fill(aliceRecoveryKey.encodedPrivateKey);
+        await dialog.locator(".mx_Dialog_primary:not([disabled])", { hasText: "Continue" }).click();
+
+        await page.locator(".mx_AuthPage").getByRole("button", { name: "Done" }).click();
+
+        // Check that our device is now cross-signed
+        await checkDeviceIsCrossSigned(app);
+
+        // Check that the current device is connected to key backup
+        await checkDeviceIsConnectedKeyBackup(page);
+    });
+
+    test("Handle incoming verification request with SAS", async ({ page, credentials, homeserver, toasts }) => {
+        await logIntoElement(page, homeserver, credentials);
+
+        /* Dismiss "Verify this device" */
+        const authPage = page.locator(".mx_AuthPage");
+        await authPage.getByRole("button", { name: "Skip verification for now" }).click();
+        await authPage.getByRole("button", { name: "I'll verify later" }).click();
+
+        await page.waitForSelector(".mx_MatrixChat");
+        const elementDeviceId = await page.evaluate(() => window.mxMatrixClientPeg.get().getDeviceId());
+
+        /* Now initiate a verification request from the *bot* device. */
+        const botVerificationRequest = await aliceBotClient.evaluateHandle(
+            async (client, { userId, deviceId }) => {
+                return client.getCrypto()!.requestDeviceVerification(userId, deviceId);
+            },
+            { userId: credentials.userId, deviceId: elementDeviceId },
+        );
+
+        /* Check the toast for the incoming request */
+        const toast = await toasts.getToast("Verification requested");
+        // it should contain the device ID of the requesting device
+        await expect(toast.getByText(`${aliceBotClient.credentials.deviceId} from `)).toBeVisible();
+        // Accept
+        await toast.getByRole("button", { name: "Verify Session" }).click();
+
+        /* Click 'Start' to start SAS verification */
+        await page.getByRole("button", { name: "Start" }).click();
+
+        /* on the bot side, wait for the verifier to exist ... */
+        const verifier = await awaitVerifier(botVerificationRequest);
+        // ... confirm ...
+        botVerificationRequest.evaluate((verificationRequest) => verificationRequest.verifier.verify());
+        // ... and then check the emoji match
+        await doTwoWaySasVerification(page, verifier);
+
+        /* And we're all done! */
+        const infoDialog = page.locator(".mx_InfoDialog");
+        await infoDialog.getByRole("button", { name: "They match" }).click();
+        await expect(
+            infoDialog.getByText(`You've successfully verified (${aliceBotClient.credentials.deviceId})!`),
+        ).toBeVisible();
+        await infoDialog.getByRole("button", { name: "Got it" }).click();
+    });
+});
+
+test.describe("User verification", () => {
+    // note that there are other tests that check user verification works in `crypto.spec.ts`.
+
+    test.use({
+        displayName: "Alice",
+        botCreateOpts: { displayName: "Bob", autoAcceptInvites: true, userIdPrefix: "bob_" },
+    });
+
+    test("can receive a verification request when there is no existing DM", async ({
+        page,
+        app,
+        bot: bob,
+        user: aliceCredentials,
+        toasts,
+    }) => {
+        await app.client.bootstrapCrossSigning(aliceCredentials);
+
+        // the other user creates a DM
+        const dmRoomId = await createDMRoom(bob, aliceCredentials.userId);
+
+        // accept the DM
+        await app.viewRoomByName("Bob");
+        await page.getByRole("button", { name: "Start chatting" }).click();
+
+        // once Alice has joined, Bob starts the verification
+        const bobVerificationRequest = await bob.evaluateHandle(
+            async (client, { dmRoomId, aliceCredentials }) => {
+                const room = client.getRoom(dmRoomId);
+                while (room.getMember(aliceCredentials.userId)?.membership !== "join") {
+                    await new Promise((resolve) => {
+                        room.once(window.matrixcs.RoomStateEvent.Members, resolve);
+                    });
+                }
+
+                return client.getCrypto().requestVerificationDM(aliceCredentials.userId, dmRoomId);
+            },
+            { dmRoomId, aliceCredentials },
+        );
+
+        // there should also be a toast
+        const toast = await toasts.getToast("Verification requested");
+        // it should contain the details of the requesting user
+        await expect(toast.getByText(`Bob (${bob.credentials.userId})`)).toBeVisible();
+        // Accept
+        await toast.getByRole("button", { name: "Verify Session" }).click();
+
+        // request verification by emoji
+        await page.locator("#mx_RightPanel").getByRole("button", { name: "Verify by emoji" }).click();
+
+        /* on the bot side, wait for the verifier to exist ... */
+        const botVerifier = await awaitVerifier(bobVerificationRequest);
+        // ... confirm ...
+        botVerifier.evaluate((verifier) => verifier.verify());
+        // ... and then check the emoji match
+        await doTwoWaySasVerification(page, botVerifier);
+
+        await page.getByRole("button", { name: "They match" }).click();
+        await expect(page.getByText("You've successfully verified Bob!")).toBeVisible();
+        await page.getByRole("button", { name: "Got it" }).click();
+    });
+});
+
+/** Extract the qrcode out of an on-screen html element */
+async function readQrCode(base: Locator) {
+    const qrCode = base.locator('[alt="QR Code"]');
+    const imageData = await qrCode.evaluate<
+        {
+            colorSpace: PredefinedColorSpace;
+            width: number;
+            height: number;
+            buffer: number[];
+        },
+        HTMLImageElement
+    >(async (img) => {
+        // draw the image on a canvas
+        const myCanvas = new OffscreenCanvas(img.width, img.height);
+        const ctx = myCanvas.getContext("2d");
+        ctx.drawImage(img, 0, 0);
+
+        // read the image data
+        const imageData = ctx.getImageData(0, 0, myCanvas.width, myCanvas.height);
+        return {
+            colorSpace: imageData.colorSpace,
+            width: imageData.width,
+            height: imageData.height,
+            buffer: [...new Uint8ClampedArray(imageData.data.buffer)],
+        };
+    });
+
+    // now we can decode the QR code.
+    const result = jsQR(new Uint8ClampedArray(imageData.buffer), imageData.width, imageData.height);
+    return new Uint8Array(result.binaryData);
+}
+
+async function createDMRoom(client: Client, userId: string): Promise<string> {
+    return client.createRoom({
+        preset: "trusted_private_chat" as Preset,
+        visibility: "private" as Visibility,
+        invite: [userId],
+        is_direct: true,
+        initial_state: [
+            {
+                type: "m.room.encryption",
+                state_key: "",
+                content: {
+                    algorithm: "m.megolm.v1.aes-sha2",
+                },
+            },
+        ],
+    });
+}
+
+/**
+ * Wait for a verifier to exist for a VerificationRequest
+ *
+ * @param botVerificationRequest
+ */
+async function awaitVerifier(botVerificationRequest: JSHandle<VerificationRequest>): Promise<JSHandle<Verifier>> {
+    return botVerificationRequest.evaluateHandle(async (verificationRequest) => {
+        while (!verificationRequest.verifier) {
+            await new Promise((r) => verificationRequest.once("change" as any, r));
+        }
+        return verificationRequest.verifier;
+    });
+}