Add flaky test labels for playwright projects (#28980)

Signed-off-by: Michael Telatynski <[email protected]>
element-hq · Jan 13, 2025 · 5a5db19 · 5a5db19
1 parent 11a8723
commit 5a5db19
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 4 deletions.
diff --git a/.github/labels.yml b/.github/labels.yml
@@ -235,6 +235,15 @@
 - name: "Z-Flaky-Test"
   description: "A test is raising false alarms"
   color: "ededed"
+- name: "Z-Flaky-Test-Chrome"
+  description: "Flaky playwright test in Chrome"
+  color: "ededed"
+- name: "Z-Flaky-Test-Firefox"
+  description: "Flaky playwright test in Firefox"
+  color: "ededed"
+- name: "Z-Flaky-Test-Webkit"
+  description: "Flaky playwright test in Webkit"
+  color: "ededed"
 - name: "Z-Flaky-Jest-Test"
   description: "A Jest test is raising false alarms"
   color: "ededed"

diff --git a/playwright/flaky-reporter.ts b/playwright/flaky-reporter.ts
@@ -25,12 +25,15 @@ type PaginationLinks = {
 };
 
 class FlakyReporter implements Reporter {
-    private flakes = new Set<string>();
+    private flakes = new Map<string, TestCase[]>();
 
     public onTestEnd(test: TestCase): void {
         const title = `${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`;
         if (test.outcome() === "flaky") {
-            this.flakes.add(title);
+            if (!this.flakes.has(title)) {
+                this.flakes.set(title, []);
+            }
+            this.flakes.get(title).push(test);
         }
     }
 
@@ -97,12 +100,14 @@ class FlakyReporter implements Reporter {
         if (!GITHUB_TOKEN) return;
 
         const issues = await this.getAllIssues();
-        for (const flake of this.flakes) {
+        for (const [flake, results] of this.flakes) {
             const title = ISSUE_TITLE_PREFIX + "`" + flake + "`";
             const existingIssue = issues.find((issue) => issue.title === title);
             const headers = { Authorization: `Bearer ${GITHUB_TOKEN}` };
             const body = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`;
 
+            const labels = [LABEL, ...results.map((test) => test.parent.project()?.name).filter(Boolean)];
+
             if (existingIssue) {
                 console.log(`Found issue ${existingIssue.number} for ${flake}, adding comment...`);
                 // Ensure that the test is open
@@ -111,6 +116,11 @@ class FlakyReporter implements Reporter {
                     headers,
                     body: JSON.stringify({ state: "open" }),
                 });
+                await fetch(`${existingIssue.url}/labels`, {
+                    method: "POST",
+                    headers,
+                    body: JSON.stringify({ labels }),
+                });
                 await fetch(`${existingIssue.url}/comments`, {
                     method: "POST",
                     headers,
@@ -124,7 +134,7 @@ class FlakyReporter implements Reporter {
                     body: JSON.stringify({
                         title,
                         body,
-                        labels: [LABEL],
+                        labels: [...labels],
                     }),
                 });
             }