Fix flaky tests

Signed-off-by: Michael Telatynski <[email protected]>

playwright/e2e/crypto/backups-mas.spec.ts

@@ -62,8 +62,7 @@ test.describe("Key backup reset from elsewhere", () => {
         await page.getByRole("textbox", { name: "Name" }).fill("test room");
         await page.getByRole("button", { name: "Create room" }).click();
 
-        // @ts-ignore - this runs in the browser scope where mxMatrixClientPeg is a thing. Here, it is not.
-        const accessToken = await page.evaluate(() => mxMatrixClientPeg.get().getAccessToken());
+        const accessToken = await page.evaluate(() => window.mxMatrixClientPeg.get().getAccessToken());
 
         const csAPI = new TestClientServerAPI(request, homeserver, accessToken);
 

playwright/e2e/login/login-consent.spec.ts

@@ -88,6 +88,11 @@ test.use({
             },
         },
     },
+    context: async ({ context, homeserver }, use) => {
+        // Restart the homeserver to wipe its in-memory db so we can reuse the same user ID without cross-signing prompts
+        await homeserver.restart();
+        await use(context);
+    },
     credentials: async ({ context, homeserver }, use) => {
         const displayName = "Dave";
         const credentials = await homeserver.registerUser(username, password, displayName);

playwright/services.ts

@@ -131,10 +131,11 @@ export const test = base.extend<{}, Services>({
         { scope: "worker" },
     ],
 
-    context: async ({ logger, context, request, homeserver }, use, testInfo) => {
+    context: async ({ logger, context, request, homeserver, mailhogClient }, use, testInfo) => {
         homeserver.setRequest(request);
         await logger.testStarted(testInfo);
         await use(context);
         await logger.testFinished(testInfo);
+        await mailhogClient.deleteAll();
     },
 });

playwright/stale-screenshot-reporter.ts

@@ -20,10 +20,13 @@ const snapshotRoot = path.join(__dirname, "snapshots");
 
 class StaleScreenshotReporter implements Reporter {
     private screenshots = new Set<string>();
+    private failing = false;
     private success = true;
 
     public onTestEnd(test: TestCase): void {
-        if (!test.ok()) return;
+        if (!test.ok()) {
+            this.failing = true;
+        }
         for (const annotation of test.annotations) {
             if (annotation.type === "_screenshot") {
                 this.screenshots.add(annotation.description);
@@ -40,6 +43,7 @@ class StaleScreenshotReporter implements Reporter {
     }
 
     public async onExit(): Promise<void> {
+        if (this.failing) return;
         const screenshotFiles = new Set(await glob(`**/*.png`, { cwd: snapshotRoot }));
         for (const screenshot of screenshotFiles) {
             if (screenshot.split("-").at(-1) !== "linux.png") {

playwright/testcontainers/mas.ts

@@ -168,6 +168,14 @@ const DEFAULT_CONFIG = {
         access_token_ttl: 300,
         compat_token_ttl: 300,
     },
+    rate_limiting: {
+        login: {
+            burst: 1000,
+        },
+        registration: {
+            burst: 1000,
+        },
+    },
 };
 
 export class MatrixAuthenticationServiceContainer extends GenericContainer {

playwright/testcontainers/synapse.ts

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait } from "testcontainers";
+import { AbstractStartedContainer, GenericContainer, RestartOptions, StartedTestContainer, Wait } from "testcontainers";
 import { APIRequestContext } from "@playwright/test";
 import crypto from "node:crypto";
 import * as YAML from "yaml";
@@ -239,6 +239,11 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements
         super(container);
     }
 
+    public restart(options?: Partial<RestartOptions>): Promise<void> {
+        this.adminToken = undefined;
+        return super.restart(options);
+    }
+
     public setRequest(request: APIRequestContext): void {
         this.request = request;
     }