Add test reporter to prevent stale screenshots (#12743)

* Split up slow Playwright tests To optimise parallelism Deals with: ``` Slow test file: read-receipts/redactions.spec.ts (5.4m) Slow test file: read-receipts/new-messages.spec.ts (3.9m) Slow test file: read-receipts/high-level.spec.ts (3.6m) Slow test file: read-receipts/editing-messages.spec.ts (3.1m) Slow test file: read-receipts/reactions.spec.ts (2.2m) Slow test file: crypto/crypto.spec.ts (2.4m) Slow test file: settings/appearance-user-settings-tab/appearance-user-settings-tab.spec.ts (1.2m) Slow test file: composer/composer.spec.ts (1.1m) Slow test file: crypto/verification.spec.ts (1.1m) ``` Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Move around snapshots Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Add test reporter to prevent stale screenshots Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix test Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Remove darwin screenshots which should not have been checked in Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix absolute vs relative path mismatch Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Revert "Remove darwin screenshots which should not have been checked in" This reverts commit 1e189977fa9ec873339fc02b2b231a314809b2d5. * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Revert "Revert "Remove darwin screenshots which should not have been checked in"" This reverts commit 5144b9b28e31ca543b2c5d02820c3f957dbd8c04. * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Remove stale screenshots Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Revert "Remove stale screenshots" This reverts commit 9beae9974557c1ffa99c2372da280bb0da407bd1. * Apply same sanitization as Playwright for file name consistency Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * add dev dep Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Remove stale screenshots Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Discard changes to playwright/flaky-reporter.ts * Update end-to-end-tests.yaml --------- Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2024-07-15 19:23:20 +01:00 · 2024-07-15 19:23:20 +01:00 · e6d9eccf1b
commit e6d9eccf1b
parent 7863de653a
21 changed files with 131 additions and 10 deletions
--- a/playwright/stale-screenshot-reporter.ts
+++ b/playwright/stale-screenshot-reporter.ts
@ -0,0 +1,74 @@
+/*
+Copyright 2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Test reporter which compares the reported screenshots vs those on disk to find stale screenshots
+ * Only intended to run from within GitHub Actions
+ */
+
+import path from "node:path";
+import { glob } from "glob";
+
+import type { Reporter, TestCase } from "@playwright/test/reporter";
+
+const snapshotRoot = path.join(__dirname, "snapshots");
+
+class StaleScreenshotReporter implements Reporter {
+    private screenshots = new Set<string>();
+    private success = true;
+
+    public onTestEnd(test: TestCase): void {
+        for (const annotation of test.annotations) {
+            if (annotation.type === "_screenshot") {
+                this.screenshots.add(annotation.description);
+            }
+        }
+    }
+
+    private error(msg: string, file: string) {
+        if (process.env.GITHUB_ACTIONS) {
+            console.log(`::error file=${file}::${msg}`);
+        }
+        console.error(msg, file);
+        this.success = false;
+    }
+
+    public async onExit(): Promise<void> {
+        const screenshotFiles = new Set(await glob(`**/*.png`, { cwd: snapshotRoot }));
+        for (const screenshot of screenshotFiles) {
+            if (screenshot.split("-").at(-1) !== "linux.png") {
+                this.error(
+                    "Found screenshot belonging to different platform, this should not be checked in",
+                    screenshot,
+                );
+            }
+        }
+        for (const screenshot of this.screenshots) {
+            screenshotFiles.delete(screenshot);
+        }
+        if (screenshotFiles.size > 0) {
+            for (const screenshot of screenshotFiles) {
+                this.error("Stale screenshot file", screenshot);
+            }
+        }
+
+        if (!this.success) {
+            process.exit(1);
+        }
+    }
+}
+
+export default StaleScreenshotReporter;