Show a progress bar while migrating from legacy crypto (#12104)

* Show a progress bar during migration of crypto data

* playwright: add new `pageWithCredentials` fixture

* Add a playwright test for migration progress

* Add documentation for `idbSave`

playwright/e2e/crypto/migration.spec.ts

@@ -0,0 +1,72 @@
+/*
+Copyright 2023-2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import path from "path";
+import { readFile } from "node:fs/promises";
+
+import { expect, test as base } from "../../element-web-test";
+
+const test = base.extend({
+    // Replace the `user` fixture with one which populates the indexeddb data before starting the app.
+    user: async ({ context, pageWithCredentials: page, credentials }, use) => {
+        await page.route(`/test_indexeddb_cryptostore_dump/*`, async (route, request) => {
+            const resourcePath = path.join(__dirname, new URL(request.url()).pathname);
+            const body = await readFile(resourcePath, { encoding: "utf-8" });
+            await route.fulfill({ body });
+        });
+        await page.goto("/test_indexeddb_cryptostore_dump/index.html");
+
+        await use(credentials);
+    },
+});
+
+test.describe("migration", function () {
+    test.use({ displayName: "Alice" });
+
+    test("Should support migration from legacy crypto", async ({ context, user, page }, workerInfo) => {
+        test.skip(workerInfo.project.name === "Legacy Crypto", "This test only works with Rust crypto.");
+        test.slow();
+
+        // We should see a migration progress bar
+        await page.getByText("Hang tight.").waitFor({ timeout: 60000 });
+
+        // When the progress bar first loads, it should have a high max (one per megolm session to import), and
+        // a relatively low value.
+        const progressBar = page.getByRole("progressbar");
+        const initialProgress = parseFloat(await progressBar.getAttribute("value"));
+        const initialMax = parseFloat(await progressBar.getAttribute("max"));
+        expect(initialMax).toBeGreaterThan(4000);
+        expect(initialProgress).toBeGreaterThanOrEqual(0);
+        expect(initialProgress).toBeLessThanOrEqual(500);
+
+        // Later, the progress should pass 50%
+        await expect
+            .poll(
+                async () => {
+                    const progressBar = page.getByRole("progressbar");
+                    return (
+                        (parseFloat(await progressBar.getAttribute("value")) * 100.0) /
+                        parseFloat(await progressBar.getAttribute("max"))
+                    );
+                },
+                { timeout: 60000 },
+            )
+            .toBeGreaterThan(50);
+
+        // Eventually, we should get a normal matrix chat
+        await page.waitForSelector(".mx_MatrixChat", { timeout: 120000 });
+    });
+});

playwright/e2e/crypto/test_indexeddb_cryptostore_dump/README.md

@@ -0,0 +1,60 @@
+# Dump of libolm indexeddb cryptostore
+
+This directory contains, in `dump.json`, a dump of a real indexeddb store from a session using
+libolm crypto.
+
+The corresponding pickle key is `+1k2Ppd7HIisUY824v7JtV3/oEE4yX0TqtmNPyhaD7o`.
+
+This directory also contains, in `index.html` and `load.js`, a page which will populate indexeddb with the data
+(and the pickle key). This can be served via a Playwright [Route](https://playwright.dev/docs/api/class-route) so as to
+populate the indexeddb before the main application loads. Note that encrypting the pickle key requires the test User ID
+and Device ID, so they must be stored in `localstorage` before loading `index.html`.
+
+## Creation of the dump file
+
+The dump was created by pasting the following into the browser console:
+
+```javascript
+async function exportIndexedDb(name) {
+    const db = await new Promise((resolve, reject) => {
+        const dbReq = indexedDB.open(name);
+        dbReq.onerror = reject;
+        dbReq.onsuccess = () => resolve(dbReq.result);
+    });
+
+    const storeNames = db.objectStoreNames;
+    const exports = {};
+    for (const store of storeNames) {
+        exports[store] = [];
+        const txn = db.transaction(store, "readonly");
+        const objectStore = txn.objectStore(store);
+        await new Promise((resolve, reject) => {
+            const cursorReq = objectStore.openCursor();
+            cursorReq.onerror = reject;
+            cursorReq.onsuccess = (event) => {
+                const cursor = event.target.result;
+                if (cursor) {
+                    const entry = { value: cursor.value };
+                    if (!objectStore.keyPath) {
+                        entry.key = cursor.key;
+                    }
+                    exports[store].push(entry);
+                    cursor.continue();
+                } else {
+                    resolve();
+                }
+            };
+        });
+    }
+    return exports;
+}
+
+window.saveAs(
+    new Blob([JSON.stringify(await exportIndexedDb("matrix-js-sdk:crypto"), null, 2)], {
+        type: "application/json;charset=utf-8",
+    }),
+    "dump.json",
+);
+```
+
+The pickle key is extracted via `mxMatrixClientPeg.get().crypto.olmDevice.pickleKey`.

playwright/e2e/crypto/test_indexeddb_cryptostore_dump/index.html

@@ -0,0 +1,6 @@
+<html>
+    <head>
+        <script src="load.js"></script>
+    </head>
+    Loading test data...
+</html>

playwright/e2e/crypto/test_indexeddb_cryptostore_dump/load.js

@@ -0,0 +1,228 @@
+/*
+Copyright 2023-2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/* Browser-side javascript to fetch the indexeddb dump file, and populate indexeddb. */
+
+/** The pickle key corresponding to the data dump. */
+const PICKLE_KEY = "+1k2Ppd7HIisUY824v7JtV3/oEE4yX0TqtmNPyhaD7o";
+
+/**
+ * Populate an IndexedDB store with the test data from this directory.
+ *
+ * @param {any} data - IndexedDB dump to import
+ * @param {string} name - Name of the IndexedDB database to create.
+ */
+async function populateStore(data, name) {
+    const req = indexedDB.open(name, 11);
+
+    const db = await new Promise((resolve, reject) => {
+        req.onupgradeneeded = (ev) => {
+            const db = req.result;
+            const oldVersion = ev.oldVersion;
+            upgradeDatabase(oldVersion, db);
+        };
+
+        req.onerror = (ev) => {
+            reject(req.error);
+        };
+
+        req.onsuccess = () => {
+            const db = req.result;
+            resolve(db);
+        };
+    });
+
+    await importData(data, db);
+
+    return db;
+}
+
+/**
+ * Create the schema for the indexed db store
+ *
+ * @param {number} oldVersion - The current version of the store.
+ * @param {IDBDatabase} db - The indexeddb database.
+ */
+function upgradeDatabase(oldVersion, db) {
+    if (oldVersion < 1) {
+        const outgoingRoomKeyRequestsStore = db.createObjectStore("outgoingRoomKeyRequests", { keyPath: "requestId" });
+        outgoingRoomKeyRequestsStore.createIndex("session", ["requestBody.room_id", "requestBody.session_id"]);
+        outgoingRoomKeyRequestsStore.createIndex("state", "state");
+    }
+
+    if (oldVersion < 2) {
+        db.createObjectStore("account");
+    }
+
+    if (oldVersion < 3) {
+        const sessionsStore = db.createObjectStore("sessions", { keyPath: ["deviceKey", "sessionId"] });
+        sessionsStore.createIndex("deviceKey", "deviceKey");
+    }
+
+    if (oldVersion < 4) {
+        db.createObjectStore("inbound_group_sessions", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 5) {
+        db.createObjectStore("device_data");
+    }
+
+    if (oldVersion < 6) {
+        db.createObjectStore("rooms");
+    }
+
+    if (oldVersion < 7) {
+        db.createObjectStore("sessions_needing_backup", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 8) {
+        db.createObjectStore("inbound_group_sessions_withheld", { keyPath: ["senderCurve25519Key", "sessionId"] });
+    }
+
+    if (oldVersion < 9) {
+        const problemsStore = db.createObjectStore("session_problems", { keyPath: ["deviceKey", "time"] });
+        problemsStore.createIndex("deviceKey", "deviceKey");
+
+        db.createObjectStore("notified_error_devices", { keyPath: ["userId", "deviceId"] });
+    }
+
+    if (oldVersion < 10) {
+        db.createObjectStore("shared_history_inbound_group_sessions", { keyPath: ["roomId"] });
+    }
+
+    if (oldVersion < 11) {
+        db.createObjectStore("parked_shared_history", { keyPath: ["roomId"] });
+    }
+}
+
+/** Do the import of data into the database
+ *
+ * @param {any} json - The data to import.
+ * @param {IDBDatabase} db - The database to import into.
+ * @returns {Promise<void>}
+ */
+async function importData(json, db) {
+    for (const [storeName, data] of Object.entries(json)) {
+        await new Promise((resolve, reject) => {
+            console.log(`Populating ${storeName} with test data`);
+            const store = db.transaction(storeName, "readwrite").objectStore(storeName);
+
+            function putEntry(idx) {
+                if (idx >= data.length) {
+                    resolve(undefined);
+                    return;
+                }
+
+                const { key, value } = data[idx];
+                try {
+                    const putReq = store.put(value, key);
+                    putReq.onsuccess = (_) => putEntry(idx + 1);
+                    putReq.onerror = (_) => reject(putReq.error);
+                } catch (e) {
+                    throw new Error(
+                        `Error populating '${storeName}' with key ${JSON.stringify(key)}, value ${JSON.stringify(
+                            value,
+                        )}: ${e}`,
+                    );
+                }
+            }
+
+            putEntry(0);
+        });
+    }
+}
+
+function getPickleAdditionalData(userId, deviceId) {
+    const additionalData = new Uint8Array(userId.length + deviceId.length + 1);
+    for (let i = 0; i < userId.length; i++) {
+        additionalData[i] = userId.charCodeAt(i);
+    }
+    additionalData[userId.length] = 124; // "|"
+    for (let i = 0; i < deviceId.length; i++) {
+        additionalData[userId.length + 1 + i] = deviceId.charCodeAt(i);
+    }
+    return additionalData;
+}
+
+/** Save an entry to the `matrix-react-sdk` indexeddb database.
+ *
+ * If `matrix-react-sdk` does not yet exist, it will be created with the correct schema.
+ *
+ * @param {String} table
+ * @param {String} key
+ * @param {String} data
+ * @returns {Promise<void>}
+ */
+async function idbSave(table, key, data) {
+    const idb = await new Promise((resolve, reject) => {
+        const request = indexedDB.open("matrix-react-sdk", 1);
+        request.onerror = reject;
+        request.onsuccess = () => {
+            resolve(request.result);
+        };
+        request.onupgradeneeded = () => {
+            const db = request.result;
+            db.createObjectStore("pickleKey");
+            db.createObjectStore("account");
+        };
+    });
+    return await new Promise((resolve, reject) => {
+        const txn = idb.transaction([table], "readwrite");
+        txn.onerror = reject;
+
+        const objectStore = txn.objectStore(table);
+        const request = objectStore.put(data, key);
+        request.onerror = reject;
+        request.onsuccess = resolve;
+    });
+}
+
+/**
+ * Save the pickle key to indexeddb, so that the app can read it.
+ *
+ * @param {String} userId  - The user's ID (used in the encryption algorithm).
+ * @param {String} deviceId  - The user's device ID (ditto).
+ * @returns {Promise<void>}
+ */
+async function savePickleKey(userId, deviceId) {
+    const itFunc = function* () {
+        const decoded = atob(PICKLE_KEY);
+        for (let i = 0; i < decoded.length; ++i) {
+            yield decoded.charCodeAt(i);
+        }
+    };
+    const decoded = Uint8Array.from(itFunc());
+
+    const cryptoKey = await crypto.subtle.generateKey({ name: "AES-GCM", length: 256 }, false, ["encrypt", "decrypt"]);
+    const iv = new Uint8Array(32);
+    crypto.getRandomValues(iv);
+
+    const additionalData = getPickleAdditionalData(userId, deviceId);
+    const encrypted = await crypto.subtle.encrypt({ name: "AES-GCM", iv, additionalData }, cryptoKey, decoded);
+
+    await idbSave("pickleKey", [userId, deviceId], { encrypted, iv, cryptoKey });
+}
+
+async function loadDump() {
+    const dump = await fetch("dump.json");
+    const indexedDbDump = await dump.json();
+    await populateStore(indexedDbDump, "matrix-js-sdk:crypto");
+    await savePickleKey(window.localStorage.getItem("mx_user_id"), window.localStorage.getItem("mx_device_id"));
+    console.log("Test data loaded; redirecting to main app");
+    window.location.replace("/");
+}
+
+loadDump();

playwright/element-web-test.ts

@@ -73,6 +73,16 @@ export const test = base.extend<
         homeserver: HomeserverInstance;
         oAuthServer: { port: number };
         credentials: CredentialsWithDisplayName;
+
+        /**
+         * The same as {@link https://playwright.dev/docs/api/class-fixtures#fixtures-page|`page`},
+         * but adds an initScript which will populate localStorage with the user's details from
+         * {@link #credentials} and {@link #homeserver}.
+         *
+         * Similar to {@link #user}, but doesn't load the app.
+         */
+        pageWithCredentials: Page;
+
         user: CredentialsWithDisplayName;
         displayName?: string;
         app: ElementAppPage;
@@ -163,7 +173,8 @@ export const test = base.extend<
         });
     },
     labsFlags: [],
-    user: async ({ page, homeserver, credentials }, use) => {
+
+    pageWithCredentials: async ({ page, homeserver, credentials }, use) => {
         await page.addInitScript(
             ({ baseUrl, credentials }) => {
                 // Seed the localStorage with the required credentials
@@ -180,9 +191,12 @@ export const test = base.extend<
             },
             { baseUrl: homeserver.config.baseUrl, credentials },
         );
+        await use(page);
+    },
+
+    user: async ({ pageWithCredentials: page, credentials }, use) => {
         await page.goto("/");
         await page.waitForSelector(".mx_MatrixChat", { timeout: 30000 });
-
         await use(credentials);
     },