Fix intermittently failing Playwright tests (#6886)

# Description of Changes
Fixes intermittently failing tests (and replaces one that wasn't useful
in its previous state) and also adds a CI check to warn if there are any
Playwright tests which failed on their first go and succeeded on
retries, to hopefully help find intermittently failing tests more
quickly and avoid them being merged in the first place.

.github/workflows/build-enterprise.yml

@@ -167,6 +167,8 @@ jobs:
           wait_for_backend
       - name: Run enterprise OAuth Playwright tests
         id: oauth-tests
+        env:
+          PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-oauth.json
         run: task e2e:enterprise -- --grep "OAuth"
       - name: Stop backend + tear down OAuth Keycloak
         if: always()
@@ -240,6 +242,8 @@ jobs:
           wait_for_backend
       - name: Run enterprise SAML Playwright tests
         id: saml-tests
+        env:
+          PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-saml.json
         run: task e2e:enterprise -- --grep "SAML"
       - name: Stop backend + tear down SAML Keycloak
         if: always()
@@ -270,6 +274,8 @@ jobs:
           wait_for_backend
       - name: Run enterprise feature Playwright tests
         id: feature-tests
+        env:
+          PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results-feature.json
         run: task e2e:enterprise -- --grep "Enterprise license"
       - name: Print backend log on failure
         if: failure()
@@ -282,6 +288,19 @@ jobs:
         run: |
           source /tmp/helpers.sh
           stop_backend
+      - name: Flag flaky tests
+        # Runs regardless of the test outcomes: a flaky test (passed on retry)
+        # leaves its step green, so this is the only place it surfaces. Merges
+        # all three phase reports (some may be absent if an earlier phase hard-
+        # failed and skipped the rest). Emits ::warning:: annotations + a job
+        # summary; never fails the job.
+        if: always()
+        working-directory: frontend
+        run: >
+          npx tsx editor/scripts/report-flaky-tests.mts
+          "${{ github.workspace }}/frontend/playwright-report/results-oauth.json"
+          "${{ github.workspace }}/frontend/playwright-report/results-saml.json"
+          "${{ github.workspace }}/frontend/playwright-report/results-feature.json"
       - name: Upload Playwright report
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.github/workflows/e2e-live.yml

@@ -62,7 +62,17 @@ jobs:
           # .test-state/playwright/coverage-pw/ for the post-process step
           # to aggregate. Chromium-only - other engines silently skip.
           PW_COVERAGE: "1"
+          PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
         run: task e2e:live
+      - name: Flag flaky tests
+        # Runs regardless of the test outcome: a flaky test (passed on retry)
+        # leaves the step green, so this is the only place it surfaces. Emits
+        # ::warning:: annotations + a job summary; never fails the job.
+        if: always()
+        working-directory: frontend
+        run: npx tsx editor/scripts/report-flaky-tests.mts "$PLAYWRIGHT_JSON_OUTPUT_FILE"
+        env:
+          PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
       - name: Generate JaCoCo report from e2e:live .exec
         if: always()
         id: live-coverage

.github/workflows/e2e-stubbed.yml

@@ -44,7 +44,18 @@ jobs:
           VITE_BUILD_FOR_PREVIEW: "1"
         run: task frontend:build
       - name: Run stubbed E2E tests (chromium)
+        env:
+          PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
         run: task e2e:stubbed -- --workers=3
+      - name: Flag flaky tests
+        # Runs regardless of the test outcome: a flaky test (passed on retry)
+        # leaves the step green, so this is the only place it surfaces. Emits
+        # ::warning:: annotations + a job summary; never fails the job.
+        if: always()
+        working-directory: frontend
+        run: npx tsx editor/scripts/report-flaky-tests.mts "$PLAYWRIGHT_JSON_OUTPUT_FILE"
+        env:
+          PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/frontend/playwright-report/results.json
       - name: Upload Playwright report
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1