diff --git a/frontend/editor/src/core/tests/helpers/test-base.ts b/frontend/editor/src/core/tests/helpers/test-base.ts index 04eda2b1b6..6d782a1406 100644 --- a/frontend/editor/src/core/tests/helpers/test-base.ts +++ b/frontend/editor/src/core/tests/helpers/test-base.ts @@ -7,27 +7,189 @@ import { test as base, expect, type Page } from "@playwright/test"; const FAILING_CONSOLE_TYPES = new Set(["error", "warning"]); /** - * Attach listeners that record any `console.error` / `console.warn` calls - * and uncaught page errors. Returns a snapshot getter; the fixture asserts - * the snapshot is empty during teardown so the failure is attributed to - * the test that produced the noise. + * Patterns ignored globally on every page. Keep this list small and + * well-justified — each entry suppresses a genuine console warning for + * every test, which means we lose detection of regressions in that + * surface. Only add things that: + * + * - fire on first page render of *every* test (so per-test + * suppression would just be ceremony), AND + * - are environmental noise (third-party CDN, dev-server quirk, + * known init-order quirk) rather than something a test could + * reasonably assert. + * + * Anything that fires only on specific flows belongs in an inline + * `expectConsoleError` / `suppressConsoleErrors` at the call site. */ -function attachConsoleErrorRecorder(page: Page): () => string[] { - const messages: string[] = []; +const GLOBAL_IGNORE_PATTERNS: RegExp[] = [ + // Stripe.js logs an HTTP warning when loaded over localhost. Third-party, + // expected in dev, no production impact. + /You may test your Stripe\.js integration over HTTP/, + // i18next's HTTP backend fails to load namespace files under Vite dev's + // `@fs/` URLs; the app falls back to embedded English copy and tests + // still pass functional assertions. + /i18next::backendConnector: loading namespace/, + // scarfTracking.firePixel() is invoked from a router effect on the first + // route render, before the useScarfTracking hook has called + // setScarfConfig(). Harmless (the pixel is a no-op on first call) but + // worth a follow-up to reorder init. See utils/scarfTracking.ts. + /\[scarfTracking\] firePixel\(\) called before setScarfConfig/, + + // ── Vite dev-server flakiness under parallel-worker load ──────────────── + // The next block suppresses the entire cascade that follows when Vite's + // dev server briefly stops accepting connections (because several workers + // hit it simultaneously). In CI we serve a pre-built dist via + // `vite preview`, where none of this happens; locally the cascade is just + // environmental noise. None of these patterns mask production-only bugs. + + // 1) Browser-level network failure for an unreachable URL. + /Failed to load resource: net::ERR_/, + // 2) Vite's lazy chunk loader sees the network failure and throws. + /Failed to fetch dynamically imported module/, + // 3) PDF.js / pdfium WASM streaming fetch trips on the same outage. + /WebAssembly compilation aborted: Network error/, + /wasm streaming compile failed/, + /failed to asynchronously prepare wasm/, + /falling back to ArrayBuffer instantiation/, + // 4) React-dom logs its own wrapper line when the lazy chunk error reaches + // a Suspense / ErrorBoundary. Suppress only this exact wrapper — real + // React errors that aren't chunk-load failures still surface elsewhere. + /The above error occurred in one of your React components/, + // 5) Our ErrorBoundary's componentDidCatch dumps ~15 supplementary + // diagnostic lines. They are useful in prod but in tests they are + // pure noise on top of whatever already failed. Match by source URL. + /\(https?:\/\/[^)]*\/src\/core\/components\/shared\/ErrorBoundary\.tsx:/, +]; + +/** + * Per-page collector for console errors / warnings / uncaught page errors. + * + * The fixture installs one of these on every page. Messages that aren't + * absorbed by an active `expectConsoleError` / `suppressConsoleErrors` + * scope are reported in fixture teardown and fail the test. + */ +class ConsoleErrorRecorder { + private readonly failed: string[] = []; + private readonly scopes: Array<{ pattern: RegExp; matched: boolean }> = []; + + record(text: string): void { + for (const pattern of GLOBAL_IGNORE_PATTERNS) { + if (pattern.test(text)) return; // documented global noise + } + for (const scope of this.scopes) { + if (scope.pattern.test(text)) { + scope.matched = true; + return; // absorbed by an active scope, not a failure + } + } + this.failed.push(text); + } + + async withScope( + pattern: RegExp, + fn: () => Promise, + requireMatch: boolean, + ): Promise { + const scope = { pattern, matched: false }; + this.scopes.push(scope); + try { + const result = await fn(); + if (requireMatch && !scope.matched) { + throw new Error( + `expectConsoleError: no console error/warning matched ${pattern} ` + + `during the scoped action`, + ); + } + return result; + } finally { + const idx = this.scopes.indexOf(scope); + if (idx >= 0) this.scopes.splice(idx, 1); + } + } + + failures(): string[] { + return this.failed; + } +} + +const recordersByPage = new WeakMap(); + +function attachConsoleErrorRecorder(page: Page): ConsoleErrorRecorder { + const recorder = new ConsoleErrorRecorder(); + recordersByPage.set(page, recorder); page.on("console", (msg) => { const type = msg.type(); if (!FAILING_CONSOLE_TYPES.has(type)) return; const { url, lineNumber, columnNumber } = msg.location(); const where = url ? ` (${url}:${lineNumber}:${columnNumber})` : ""; - messages.push(`[console.${type}] ${msg.text()}${where}`); + recorder.record(`[console.${type}] ${msg.text()}${where}`); }); page.on("pageerror", (err) => { - messages.push(`[pageerror] ${err.message}`); + recorder.record(`[pageerror] ${err.message}`); }); - return () => messages; + return recorder; +} + +function getRecorder(page: Page, caller: string): ConsoleErrorRecorder { + const recorder = recordersByPage.get(page); + if (!recorder) { + throw new Error( + `${caller} requires the \`test\` exported from ` + + `\`@app/tests/helpers/test-base\` (or stub-test-base). ` + + `Are you importing \`test\` directly from "@playwright/test"?`, + ); + } + return recorder; +} + +/** + * Run `fn` and *require* at least one console error / warning / page error + * that matches `pattern` to occur during it. Matching messages are + * absorbed (they don't fail the test); if none match, this throws. + * + * Use when a test deliberately exercises an error path and the error + * surfaces in the console: + * + * await expectConsoleError(page, /Validation failed/, async () => { + * await page.getByRole("button", { name: "Submit" }).click(); + * await expect(page.getByRole("alert")).toBeVisible(); + * }); + * + * The scope only covers messages emitted while `fn` is awaiting, so + * remember to `await` any UI assertion that the error has surfaced + * *inside* the callback rather than after it returns. + */ +export async function expectConsoleError( + page: Page, + pattern: RegExp, + fn: () => Promise, +): Promise { + return getRecorder(page, "expectConsoleError").withScope(pattern, fn, true); +} + +/** + * Run `fn` and silently absorb any console errors / warnings / page + * errors matching `pattern`, without asserting that one occurred. Use + * sparingly — `expectConsoleError` is preferred because it also verifies + * the error path actually fires. + * + * await suppressConsoleErrors(page, /MUI Grid v1 deprecated/, async () => { + * await page.getByRole("button", { name: "Open settings" }).click(); + * }); + */ +export async function suppressConsoleErrors( + page: Page, + pattern: RegExp, + fn: () => Promise, +): Promise { + return getRecorder(page, "suppressConsoleErrors").withScope( + pattern, + fn, + false, + ); } /** @@ -38,14 +200,16 @@ function attachConsoleErrorRecorder(page: Page): () => string[] { * 1. The cookie-consent cookie is seeded before any navigation so the * `#cc-main` banner never renders and never intercepts clicks. * 2. Console errors/warnings and uncaught page errors are captured. - * If any occur during the test, the fixture throws during teardown - * and the test fails. + * If any unhandled message appears during the test, the fixture + * throws during teardown and the test fails. Tests that legitimately + * produce errors should wrap the offending step in + * `expectConsoleError(page, /pattern/, async () => { ... })`. * * Usage: import { test, expect } from '@app/tests/helpers/test-base'; */ export const test = base.extend({ page: async ({ page }, use) => { - const snapshot = attachConsoleErrorRecorder(page); + const recorder = attachConsoleErrorRecorder(page); // Set the cookie consent cookie before any navigation so the banner // never appears. The cookieconsent library (orestbida/cookieconsent) @@ -68,11 +232,13 @@ export const test = base.extend({ await use(page); - const messages = snapshot(); - if (messages.length > 0) { + const failures = recorder.failures(); + if (failures.length > 0) { throw new Error( - `Test produced ${messages.length} console error(s)/warning(s):\n` + - messages.map((m) => ` ${m}`).join("\n"), + `Test produced ${failures.length} unhandled console error(s)/warning(s):\n` + + failures.map((m) => ` ${m}`).join("\n") + + `\n\nIf any of these are expected, wrap the action in ` + + `expectConsoleError(page, /pattern/, async () => { ... }).`, ); } },