From 9c8be25ff0fdc6ce996cefda5a7641798385cc42 Mon Sep 17 00:00:00 2001
From: James Brunton <jbrunton96@gmail.com>
Date: Fri, 29 May 2026 16:28:00 +0100
Subject: [PATCH] Fail Playwright tests on warnings/errors being thrown to the
 console

---
 .../enterprise/license-and-features.spec.ts   |  2 +-
 .../enterprise/oauth-keycloak-login.spec.ts   |  2 +-
 .../enterprise/saml-keycloak-login.spec.ts    |  2 +-
 .../src/core/tests/helpers/stub-test-base.ts  |  6 +--
 .../src/core/tests/helpers/test-base.ts       | 54 +++++++++++++++++--
 .../core/tests/stubbed/audit-log-ui.spec.ts   |  3 +-
 .../stubbed/certificate-validation.spec.ts    |  3 +-
 .../src/core/tests/stubbed/compare.spec.ts    |  3 +-
 .../src/core/tests/stubbed/convert.spec.ts    |  3 +-
 .../stubbed/encrypted-pdf-unlock.spec.ts      |  3 +-
 .../tests/stubbed/first-login-modal.spec.ts   |  3 +-
 .../core/tests/stubbed/license-states.spec.ts |  3 +-
 .../stubbed/premium-feature-gates.spec.ts     |  3 +-
 .../tests/stubbed/saas-backend-smoke.spec.ts  |  3 +-
 .../src/core/tests/stubbed/teams-ui.spec.ts   |  3 +-
 15 files changed, 76 insertions(+), 20 deletions(-)

diff --git a/frontend/editor/src/core/tests/enterprise/license-and-features.spec.ts b/frontend/editor/src/core/tests/enterprise/license-and-features.spec.ts
index c7f9bc367a..8f00f43354 100644
--- a/frontend/editor/src/core/tests/enterprise/license-and-features.spec.ts
+++ b/frontend/editor/src/core/tests/enterprise/license-and-features.spec.ts
@@ -1,4 +1,4 @@
-import { test, expect } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
 import { ensureCookieConsent } from "@app/tests/helpers/login";
 import { bypassOnboarding } from "@app/tests/helpers/api-stubs";
 import { openSettings } from "@app/tests/helpers/ui-helpers";
diff --git a/frontend/editor/src/core/tests/enterprise/oauth-keycloak-login.spec.ts b/frontend/editor/src/core/tests/enterprise/oauth-keycloak-login.spec.ts
index eba8995227..7cbe59a59f 100644
--- a/frontend/editor/src/core/tests/enterprise/oauth-keycloak-login.spec.ts
+++ b/frontend/editor/src/core/tests/enterprise/oauth-keycloak-login.spec.ts
@@ -1,4 +1,4 @@
-import { test, expect } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
 import { ensureCookieConsent } from "@app/tests/helpers/login";
 import { bypassOnboarding } from "@app/tests/helpers/api-stubs";
 
diff --git a/frontend/editor/src/core/tests/enterprise/saml-keycloak-login.spec.ts b/frontend/editor/src/core/tests/enterprise/saml-keycloak-login.spec.ts
index 416b68bfed..2a4425c3b8 100644
--- a/frontend/editor/src/core/tests/enterprise/saml-keycloak-login.spec.ts
+++ b/frontend/editor/src/core/tests/enterprise/saml-keycloak-login.spec.ts
@@ -1,4 +1,4 @@
-import { test, expect } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
 import { ensureCookieConsent } from "@app/tests/helpers/login";
 import { bypassOnboarding } from "@app/tests/helpers/api-stubs";
 
diff --git a/frontend/editor/src/core/tests/helpers/stub-test-base.ts b/frontend/editor/src/core/tests/helpers/stub-test-base.ts
index 5752108d42..8b3834c7b9 100644
--- a/frontend/editor/src/core/tests/helpers/stub-test-base.ts
+++ b/frontend/editor/src/core/tests/helpers/stub-test-base.ts
@@ -1,8 +1,7 @@
-import { test as base, expect } from "@playwright/test";
+import { test as base, expect } from "@app/tests/helpers/test-base";
 import {
   bypassOnboarding,
   mockAppApis,
-  seedCookieConsent,
   skipOnboarding,
   type MockAppApiOptions,
 } from "@app/tests/helpers/api-stubs";
@@ -57,7 +56,8 @@ export const test = base.extend<StubFixtures>({
   seedJwt: [false, { option: true }],
 
   page: async ({ page, stubOptions, autoGoto, seedJwt }, use) => {
-    await seedCookieConsent(page);
+    // `page` comes from test-base, which has already seeded cookie consent
+    // and attached the console-error recorder before any navigation runs.
     if (seedJwt) {
       // Logged-in users hit the orchestrator path that surfaces the
       // analytics opt-in / MFA prompts — use the stronger bypass-all flag
diff --git a/frontend/editor/src/core/tests/helpers/test-base.ts b/frontend/editor/src/core/tests/helpers/test-base.ts
index e9ff8caa8d..04eda2b1b6 100644
--- a/frontend/editor/src/core/tests/helpers/test-base.ts
+++ b/frontend/editor/src/core/tests/helpers/test-base.ts
@@ -1,14 +1,52 @@
-import { test as base, expect } from "@playwright/test";
+import { test as base, expect, type Page } from "@playwright/test";
 
 /**
- * Custom test fixture that auto-dismisses the cookie consent banner
- * before every test. The banner (#cc-main) overlays the page and
- * intercepts pointer events, causing click timeouts across all tests.
+ * Console message types that should fail the test if they appear.
+ * `console.error()` -> type "error", `console.warn()` -> type "warning".
+ */
+const FAILING_CONSOLE_TYPES = new Set(["error", "warning"]);
+
+/**
+ * Attach listeners that record any `console.error` / `console.warn` calls
+ * and uncaught page errors. Returns a snapshot getter; the fixture asserts
+ * the snapshot is empty during teardown so the failure is attributed to
+ * the test that produced the noise.
+ */
+function attachConsoleErrorRecorder(page: Page): () => string[] {
+  const messages: string[] = [];
+
+  page.on("console", (msg) => {
+    const type = msg.type();
+    if (!FAILING_CONSOLE_TYPES.has(type)) return;
+    const { url, lineNumber, columnNumber } = msg.location();
+    const where = url ? ` (${url}:${lineNumber}:${columnNumber})` : "";
+    messages.push(`[console.${type}] ${msg.text()}${where}`);
+  });
+
+  page.on("pageerror", (err) => {
+    messages.push(`[pageerror] ${err.message}`);
+  });
+
+  return () => messages;
+}
+
+/**
+ * Custom test fixture shared across all Playwright suites. Two things
+ * happen for every test that uses this base (directly or transitively
+ * via `stub-test-base.ts`):
+ *
+ *   1. The cookie-consent cookie is seeded before any navigation so the
+ *      `#cc-main` banner never renders and never intercepts clicks.
+ *   2. Console errors/warnings and uncaught page errors are captured.
+ *      If any occur during the test, the fixture throws during teardown
+ *      and the test fails.
  *
  * Usage: import { test, expect } from '@app/tests/helpers/test-base';
  */
 export const test = base.extend({
   page: async ({ page }, use) => {
+    const snapshot = attachConsoleErrorRecorder(page);
+
     // Set the cookie consent cookie before any navigation so the banner
     // never appears. The cookieconsent library (orestbida/cookieconsent)
     // reads this cookie on init and skips the banner if consent exists.
@@ -29,6 +67,14 @@ export const test = base.extend({
     ]);
 
     await use(page);
+
+    const messages = snapshot();
+    if (messages.length > 0) {
+      throw new Error(
+        `Test produced ${messages.length} console error(s)/warning(s):\n` +
+          messages.map((m) => `  ${m}`).join("\n"),
+      );
+    }
   },
 });
 
diff --git a/frontend/editor/src/core/tests/stubbed/audit-log-ui.spec.ts b/frontend/editor/src/core/tests/stubbed/audit-log-ui.spec.ts
index 0e65631f4a..bb3dc0a371 100644
--- a/frontend/editor/src/core/tests/stubbed/audit-log-ui.spec.ts
+++ b/frontend/editor/src/core/tests/stubbed/audit-log-ui.spec.ts
@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import {
   bypassOnboarding,
   mockAppApis,
diff --git a/frontend/editor/src/core/tests/stubbed/certificate-validation.spec.ts b/frontend/editor/src/core/tests/stubbed/certificate-validation.spec.ts
index 1f8886bab4..cbd3962fbe 100644
--- a/frontend/editor/src/core/tests/stubbed/certificate-validation.spec.ts
+++ b/frontend/editor/src/core/tests/stubbed/certificate-validation.spec.ts
@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import path from "path";
 
 // ---------------------------------------------------------------------------
diff --git a/frontend/editor/src/core/tests/stubbed/compare.spec.ts b/frontend/editor/src/core/tests/stubbed/compare.spec.ts
index 6bcaaee538..4ca024f9a4 100644
--- a/frontend/editor/src/core/tests/stubbed/compare.spec.ts
+++ b/frontend/editor/src/core/tests/stubbed/compare.spec.ts
@@ -18,7 +18,8 @@
  * The Vite dev server must be running (handled by playwright.config.ts webServer).
  */
 
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import path from "path";
 import { mockAppApis } from "@app/tests/helpers/api-stubs";
 
diff --git a/frontend/editor/src/core/tests/stubbed/convert.spec.ts b/frontend/editor/src/core/tests/stubbed/convert.spec.ts
index cd6a60884a..4fb7646b62 100644
--- a/frontend/editor/src/core/tests/stubbed/convert.spec.ts
+++ b/frontend/editor/src/core/tests/stubbed/convert.spec.ts
@@ -5,7 +5,8 @@
  * The Vite dev server must be running (handled by playwright.config.ts webServer).
  */
 
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import path from "path";
 import { mockAppApis } from "@app/tests/helpers/api-stubs";
 
diff --git a/frontend/editor/src/core/tests/stubbed/encrypted-pdf-unlock.spec.ts b/frontend/editor/src/core/tests/stubbed/encrypted-pdf-unlock.spec.ts
index 7e220a3bda..a2670cf9fb 100644
--- a/frontend/editor/src/core/tests/stubbed/encrypted-pdf-unlock.spec.ts
+++ b/frontend/editor/src/core/tests/stubbed/encrypted-pdf-unlock.spec.ts
@@ -17,7 +17,8 @@
  * unlock-all-wrong-password - all transitively covered or low-value.
  */
 
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import path from "path";
 import fs from "fs";
 import { mockAppApis } from "@app/tests/helpers/api-stubs";
diff --git a/frontend/editor/src/core/tests/stubbed/first-login-modal.spec.ts b/frontend/editor/src/core/tests/stubbed/first-login-modal.spec.ts
index b00c84b364..673f842632 100644
--- a/frontend/editor/src/core/tests/stubbed/first-login-modal.spec.ts
+++ b/frontend/editor/src/core/tests/stubbed/first-login-modal.spec.ts
@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import { mockAppApis, seedCookieConsent } from "@app/tests/helpers/api-stubs";
 
 /**
diff --git a/frontend/editor/src/core/tests/stubbed/license-states.spec.ts b/frontend/editor/src/core/tests/stubbed/license-states.spec.ts
index ab9256d50b..2b31d502b9 100644
--- a/frontend/editor/src/core/tests/stubbed/license-states.spec.ts
+++ b/frontend/editor/src/core/tests/stubbed/license-states.spec.ts
@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import {
   bypassOnboarding,
   mockAppApis,
diff --git a/frontend/editor/src/core/tests/stubbed/premium-feature-gates.spec.ts b/frontend/editor/src/core/tests/stubbed/premium-feature-gates.spec.ts
index 5c2f4a00a9..28ad44b96a 100644
--- a/frontend/editor/src/core/tests/stubbed/premium-feature-gates.spec.ts
+++ b/frontend/editor/src/core/tests/stubbed/premium-feature-gates.spec.ts
@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import {
   bypassOnboarding,
   mockAppApis,
diff --git a/frontend/editor/src/core/tests/stubbed/saas-backend-smoke.spec.ts b/frontend/editor/src/core/tests/stubbed/saas-backend-smoke.spec.ts
index 7cc69add53..2bcd72601c 100644
--- a/frontend/editor/src/core/tests/stubbed/saas-backend-smoke.spec.ts
+++ b/frontend/editor/src/core/tests/stubbed/saas-backend-smoke.spec.ts
@@ -6,7 +6,8 @@
  *   STIRLING_FLAVOR=saas ./gradlew :stirling-pdf:bootRun --args="--server.port=18083 --spring.profiles.include=dev"
  *   STIRLING_SAAS_URL=http://localhost:18083 npx playwright test --project=stubbed saas-backend-smoke
  */
-import { test, expect, request } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import { request } from "@playwright/test";
 
 const SAAS_URL = process.env.STIRLING_SAAS_URL ?? "http://localhost:18083";
 
diff --git a/frontend/editor/src/core/tests/stubbed/teams-ui.spec.ts b/frontend/editor/src/core/tests/stubbed/teams-ui.spec.ts
index 51ce8e2171..80c8a98895 100644
--- a/frontend/editor/src/core/tests/stubbed/teams-ui.spec.ts
+++ b/frontend/editor/src/core/tests/stubbed/teams-ui.spec.ts
@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import {
   bypassOnboarding,
   mockAppApis,