Fail Playwright tests on warnings/errors being thrown to the console

2026-05-29 16:28:00 +01:00
parent 83ea07ed6a
commit 9c8be25ff0
15 changed files with 76 additions and 20 deletions
@@ -1,4 +1,4 @@
-import { test, expect } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
 import { ensureCookieConsent } from "@app/tests/helpers/login";
 import { bypassOnboarding } from "@app/tests/helpers/api-stubs";
 import { openSettings } from "@app/tests/helpers/ui-helpers";
@@ -1,4 +1,4 @@
-import { test, expect } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
 import { ensureCookieConsent } from "@app/tests/helpers/login";
 import { bypassOnboarding } from "@app/tests/helpers/api-stubs";

@@ -1,4 +1,4 @@
-import { test, expect } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
 import { ensureCookieConsent } from "@app/tests/helpers/login";
 import { bypassOnboarding } from "@app/tests/helpers/api-stubs";

@@ -1,8 +1,7 @@
-import { test as base, expect } from "@playwright/test";
+import { test as base, expect } from "@app/tests/helpers/test-base";
 import {
  bypassOnboarding,
  mockAppApis,
-  seedCookieConsent,
  skipOnboarding,
  type MockAppApiOptions,
 } from "@app/tests/helpers/api-stubs";
@@ -57,7 +56,8 @@ export const test = base.extend<StubFixtures>({
  seedJwt: [false, { option: true }],

  page: async ({ page, stubOptions, autoGoto, seedJwt }, use) => {
-    await seedCookieConsent(page);
+    // `page` comes from test-base, which has already seeded cookie consent
+    // and attached the console-error recorder before any navigation runs.
    if (seedJwt) {
      // Logged-in users hit the orchestrator path that surfaces the
      // analytics opt-in / MFA prompts — use the stronger bypass-all flag
@@ -1,14 +1,52 @@
-import { test as base, expect } from "@playwright/test";
+import { test as base, expect, type Page } from "@playwright/test";

 /**
- * Custom test fixture that auto-dismisses the cookie consent banner
- * before every test. The banner (#cc-main) overlays the page and
- * intercepts pointer events, causing click timeouts across all tests.
+ * Console message types that should fail the test if they appear.
+ * `console.error()` -> type "error", `console.warn()` -> type "warning".
+ */
+const FAILING_CONSOLE_TYPES = new Set(["error", "warning"]);
+
+/**
+ * Attach listeners that record any `console.error` / `console.warn` calls
+ * and uncaught page errors. Returns a snapshot getter; the fixture asserts
+ * the snapshot is empty during teardown so the failure is attributed to
+ * the test that produced the noise.
+ */
+function attachConsoleErrorRecorder(page: Page): () => string[] {
+  const messages: string[] = [];
+
+  page.on("console", (msg) => {
+    const type = msg.type();
+    if (!FAILING_CONSOLE_TYPES.has(type)) return;
+    const { url, lineNumber, columnNumber } = msg.location();
+    const where = url ? ` (${url}:${lineNumber}:${columnNumber})` : "";
+    messages.push(`[console.${type}] ${msg.text()}${where}`);
+  });
+
+  page.on("pageerror", (err) => {
+    messages.push(`[pageerror] ${err.message}`);
+  });
+
+  return () => messages;
+}
+
+/**
+ * Custom test fixture shared across all Playwright suites. Two things
+ * happen for every test that uses this base (directly or transitively
+ * via `stub-test-base.ts`):
+ *
+ *   1. The cookie-consent cookie is seeded before any navigation so the
+ *      `#cc-main` banner never renders and never intercepts clicks.
+ *   2. Console errors/warnings and uncaught page errors are captured.
+ *      If any occur during the test, the fixture throws during teardown
+ *      and the test fails.
 *
 * Usage: import { test, expect } from '@app/tests/helpers/test-base';
 */
 export const test = base.extend({
  page: async ({ page }, use) => {
+    const snapshot = attachConsoleErrorRecorder(page);
+
    // Set the cookie consent cookie before any navigation so the banner
    // never appears. The cookieconsent library (orestbida/cookieconsent)
    // reads this cookie on init and skips the banner if consent exists.
@@ -29,6 +67,14 @@ export const test = base.extend({
    ]);

    await use(page);
+
+    const messages = snapshot();
+    if (messages.length > 0) {
+      throw new Error(
+        `Test produced ${messages.length} console error(s)/warning(s):\n` +
+          messages.map((m) => `  ${m}`).join("\n"),
+      );
+    }
  },
 });

@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import {
  bypassOnboarding,
  mockAppApis,
@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import path from "path";

 // ---------------------------------------------------------------------------
@@ -18,7 +18,8 @@
 * The Vite dev server must be running (handled by playwright.config.ts webServer).
 */

-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import path from "path";
 import { mockAppApis } from "@app/tests/helpers/api-stubs";

@@ -5,7 +5,8 @@
 * The Vite dev server must be running (handled by playwright.config.ts webServer).
 */

-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import path from "path";
 import { mockAppApis } from "@app/tests/helpers/api-stubs";

@@ -17,7 +17,8 @@
 * unlock-all-wrong-password - all transitively covered or low-value.
 */

-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import path from "path";
 import fs from "fs";
 import { mockAppApis } from "@app/tests/helpers/api-stubs";
@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import { mockAppApis, seedCookieConsent } from "@app/tests/helpers/api-stubs";

 /**
@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import {
  bypassOnboarding,
  mockAppApis,
@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import {
  bypassOnboarding,
  mockAppApis,
@@ -6,7 +6,8 @@
 *   STIRLING_FLAVOR=saas ./gradlew :stirling-pdf:bootRun --args="--server.port=18083 --spring.profiles.include=dev"
 *   STIRLING_SAAS_URL=http://localhost:18083 npx playwright test --project=stubbed saas-backend-smoke
 */
-import { test, expect, request } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import { request } from "@playwright/test";

 const SAAS_URL = process.env.STIRLING_SAAS_URL ?? "http://localhost:18083";

@@ -1,4 +1,5 @@
-import { test, expect, type Page } from "@playwright/test";
+import { test, expect } from "@app/tests/helpers/test-base";
+import type { Page } from "@playwright/test";
 import {
  bypassOnboarding,
  mockAppApis,