chore: update e2e test configs to better work with CI (#1458)

* Also adds a helpful test:e2e:ci script to test playwright tests inside of a docker image with a clean build to more closely resemble running in CI
* Upgrades playwright version

Author: teeohhem

.github/workflows/main.yml

@@ -103,7 +103,7 @@ jobs:
     runs-on: ubuntu-24.04
     timeout-minutes: 15
     container:
-      image: mcr.microsoft.com/playwright:v1.55.0-jammy
+      image: mcr.microsoft.com/playwright:v1.57.0-jammy
     permissions:
       contents: read
       pull-requests: write

agent_docs/development.md

@@ -12,10 +12,14 @@ yarn dev
 
 ## Key Development Scripts
 
-- `yarn app:dev`: Start API, frontend, alerts task, and common-utils in watch mode
+- `yarn app:dev`: Start API, frontend, alerts task, and common-utils in watch
+  mode
 - `yarn lint`: Run linting across all packages
 - `yarn dev:int`: Run integration tests in watch mode
 - `yarn dev:unit`: Run unit tests in watch mode (per package)
+- `yarn test:e2e`: Run Playwright E2E tests (in `packages/app`)
+- `yarn test:e2e:ci`: Run Playwright E2E tests in CI Docker environment (in
+  `packages/app`)
 
 ## Environment Configuration
 
@@ -30,12 +34,14 @@ yarn dev
 - **Unit Tests**: Jest with TypeScript support
 - **Integration Tests**: Jest with database fixtures
 - **Frontend Testing**: React Testing Library + Jest
-- **E2E Testing**: Custom smoke tests with BATS
+- **E2E Testing**: Playwright (frontend) and Custom smoke tests with BATS
+  (ingestion)
 
 ### Testing Patterns
 
 - **TDD Approach**: Write tests before implementation for new features
-- **Test organization**: Tests co-located with source files in `__tests__/` directories
+- **Test organization**: Tests co-located with source files in `__tests__/`
+  directories
 - **Mocking**: MSW for API mocking in frontend tests
 - **Database testing**: Isolated test databases with fixtures
 
@@ -52,6 +58,7 @@ yarn dev:int
 ```
 
 **CI Testing Notes:**
+
 - Uses separate Docker Compose configuration optimized for CI
 - Isolated test environment with `-p int` project name
 - Includes all necessary services (ClickHouse, MongoDB, OTel Collector)
@@ -79,6 +86,7 @@ yarn dev:int
 ### Pre-commit Hooks
 
 The project uses Husky + lint-staged to automatically run:
+
 - Prettier for formatting
 - ESLint for linting
 - API doc generation (for external API changes)
@@ -101,11 +109,11 @@ yarn run lint
 
 ## File Locations Quick Reference
 
-- **Config**: `packages/api/src/config.ts`, `packages/app/next.config.mjs`, `docker-compose.dev.yml`
+- **Config**: `packages/api/src/config.ts`, `packages/app/next.config.mjs`,
+  `docker-compose.dev.yml`
 - **Models**: `packages/api/src/models/`
 - **API Routes**: `packages/api/src/routers/`
 - **Controllers**: `packages/api/src/controllers/`
 - **Pages**: `packages/app/pages/`
 - **Components**: `packages/app/src/`
 - **Shared Utils**: `packages/common-utils/src/`
-

packages/app/package.json

@@ -20,6 +20,7 @@
     "test:e2e": "playwright test",
     "test:e2e:ui": "playwright test --ui",
     "test:e2e:debug": "playwright test --debug",
+    "test:e2e:ci": "../../scripts/test-e2e-ci.sh",
     "storybook": "storybook dev -p 6006",
     "storybook:build": "storybook build",
     "knip": "knip"
@@ -107,7 +108,7 @@
     "@chromatic-com/storybook": "^4.1.3",
     "@hookform/devtools": "^4.3.1",
     "@jedmao/location": "^3.0.0",
-    "@playwright/test": "^1.47.0",
+    "@playwright/test": "^1.57.0",
     "@storybook/addon-docs": "^10.1.4",
     "@storybook/addon-links": "^10.1.4",
     "@storybook/addon-styling-webpack": "^3.0.0",

packages/app/playwright.config.ts

@@ -19,7 +19,9 @@ export default defineConfig({
   reporter: [
     ['html'],
     ['json', { outputFile: 'test-results/results.json' }],
-    ...(process.env.CI ? [['github', {}] as const] : []),
+    ...(process.env.CI
+      ? [['github'] as const, ['list'] as const]
+      : [['list'] as const]),
   ],
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
@@ -34,7 +36,7 @@ export default defineConfig({
   },
 
   /* Global test timeout - CI needs more time than local */
-  timeout: 45 * 1000,
+  timeout: 60 * 1000,
 
   /* Configure projects for different test environments */
   projects: [
@@ -48,8 +50,9 @@ export default defineConfig({
 
   /* Run your local dev server before starting the tests */
   webServer: {
-    command:
-      'NEXT_PUBLIC_IS_LOCAL_MODE=true NEXT_TELEMETRY_DISABLED=1 PORT=8081 yarn run dev',
+    command: process.env.CI
+      ? 'NEXT_PUBLIC_IS_LOCAL_MODE=true yarn build && NEXT_PUBLIC_IS_LOCAL_MODE=true PORT=8081 yarn start'
+      : 'NEXT_PUBLIC_IS_LOCAL_MODE=true NEXT_TELEMETRY_DISABLED=1 PORT=8081 yarn run dev',
     port: 8081,
     reuseExistingServer: !process.env.CI,
     timeout: 180 * 1000,

packages/app/tests/e2e/README.md

@@ -1,6 +1,8 @@
 # End-to-End Testing
 
-This directory contains Playwright-based end-to-end tests for the HyperDX application. The tests are organized into core functionality and feature-specific test suites.
+This directory contains Playwright-based end-to-end tests for the HyperDX
+application. The tests are organized into core functionality and
+feature-specific test suites.
 
 ## Prerequisites
 
@@ -47,7 +49,9 @@ cd packages/app && yarn test:e2e --grep "@dashboard"
 
 ### Local Mode vs Full Server
 
-Tests tagged with `@local-mode` can run against the local development server without external dependencies. The test configuration automatically starts a local development server with `NEXT_PUBLIC_IS_LOCAL_MODE=true`.
+Tests tagged with `@local-mode` can run against the local development server
+without external dependencies. The test configuration automatically starts a
+local development server with `NEXT_PUBLIC_IS_LOCAL_MODE=true`.
 
 ## Test Organization
 
@@ -87,6 +91,15 @@ Run tests in debug mode with browser developer tools:
 yarn test:e2e:debug
 ```
 
+### CI Mode
+
+Run tests in ci mode, which runs it in a docker container and environment
+similar to how it runs inside of Github Actions
+
+```bash
+yarn test:e2e:ci
+```
+
 ### Single Test Debugging
 
 To debug a specific test file:
@@ -114,6 +127,7 @@ yarn playwright show-report
 ```
 
 The report includes:
+
 - Test execution timeline
 - Screenshots of failures
 - Video recordings of failed tests
@@ -122,6 +136,7 @@ The report includes:
 ### Test Results
 
 Test artifacts are stored in:
+
 - `test-results/` - Screenshots, videos, and traces for failed tests
 - `playwright-report/` - HTML report files
 
@@ -138,7 +153,8 @@ yarn playwright show-trace test-results/[test-name]/trace.zip
 The test configuration is defined in `playwright.config.ts`:
 
 - **Base URL**: `http://localhost:8080` (configurable via `PLAYWRIGHT_BASE_URL`)
-- **Test Timeout**: 60 seconds (increased from default 30s to reduce flaky test failures)
+- **Test Timeout**: 60 seconds (increased from default 30s to reduce flaky test
+  failures)
 - **Retries**: 1 retry locally, 2 on CI
 - **Workers**: Undefined (uses Playwright defaults)
 - **Screenshots**: Captured on failure only
@@ -152,6 +168,7 @@ The test configuration is defined in `playwright.config.ts`:
 ### Writing Tests
 
 Tests use the extended base test from `utils/base-test.ts` which provides:
+
 - Automatic handling of connection/sources
 - Tanstack Query devtools management
 - Network idle waiting after navigation
@@ -169,21 +186,25 @@ Tests use the extended base test from `utils/base-test.ts` which provides:
 ### Server Connection Issues
 
 If tests fail with connection errors:
+
 1. Ensure no other services are running on port 8080
 2. Check that the development server starts successfully
 3. Verify environment variables are properly configured
 
 ### Flaky Tests
 
 For intermittent failures:
+
 1. Check the HTML report for timing issues
 2. Review network logs for failed requests
-3. Consider if individual test steps need longer wait times (global timeout is now 60s)
+3. Consider if individual test steps need longer wait times (global timeout is
+   now 60s)
 4. Use the trace viewer to analyze test execution
 
 ### CI/CD Integration
 
 Tests are configured to run in CI environments with:
+
 - 60-second test timeout (same as local)
 - Multiple retry attempts (2 retries on CI vs 1 locally)
 - Artifact collection for failed tests

scripts/test-e2e-ci.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Run E2E tests in Docker with exact CI environment
+# This replicates the GitHub Actions CI environment locally for debugging
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+echo "Running E2E tests in CI Docker environment..."
+echo "Repository root: $REPO_ROOT"
+
+docker run --rm \
+  -v "$REPO_ROOT:/workspace" \
+  -w /workspace \
+  -e CI=true \
+  mcr.microsoft.com/playwright:v1.57.0-jammy \
+  bash -c '
+    # Clean all build artifacts and dependencies
+    rm -rf packages/app/.next packages/common-utils/dist node_modules packages/*/node_modules .yarn/cache
+    
+    # Fresh install
+    corepack enable
+    yarn install
+    
+    # Build in production mode
+    npx nx run-many -t ci:build
+    
+    # Run tests
+    cd packages/app
+    yarn test:e2e
+    
+    # Fix permissions so host can read results
+    chmod -R 777 test-results playwright-report 2>/dev/null || true
+  '
+
+echo "Done! Check packages/app/test-results/ for results"

yarn.lock

@@ -4268,7 +4268,7 @@ __metadata:
     "@mantine/notifications": "npm:^7.17.8"
     "@mantine/spotlight": "npm:^7.17.8"
     "@microsoft/fetch-event-source": "npm:^2.0.1"
-    "@playwright/test": "npm:^1.47.0"
+    "@playwright/test": "npm:^1.57.0"
     "@storybook/addon-docs": "npm:^10.1.4"
     "@storybook/addon-links": "npm:^10.1.4"
     "@storybook/addon-styling-webpack": "npm:^3.0.0"
@@ -7719,14 +7719,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@playwright/test@npm:^1.47.0":
-  version: 1.55.0
-  resolution: "@playwright/test@npm:1.55.0"
+"@playwright/test@npm:^1.57.0":
+  version: 1.57.0
+  resolution: "@playwright/test@npm:1.57.0"
   dependencies:
-    playwright: "npm:1.55.0"
+    playwright: "npm:1.57.0"
   bin:
     playwright: cli.js
-  checksum: 10c0/e68b59cd8271f1b57c0649fc0562ab2d5f6bba8c3653dd7bd52ca1338dc380fde34588d0254e3cd3f0f2b20af04a80dfb080419ceb7475990bb2fc4d8c474984
+  checksum: 10c0/35ba4b28be72bf0a53e33dbb11c6cff848fb9a37f49e893ce63a90675b5291ec29a1ba82c8a3b043abaead129400f0589623e9ace2e6a1c8eaa409721ecc3774
   languageName: node
   linkType: hard
 
@@ -22637,27 +22637,27 @@ __metadata:
   languageName: node
   linkType: hard
 
-"playwright-core@npm:1.55.0":
-  version: 1.55.0
-  resolution: "playwright-core@npm:1.55.0"
+"playwright-core@npm:1.57.0":
+  version: 1.57.0
+  resolution: "playwright-core@npm:1.57.0"
   bin:
     playwright-core: cli.js
-  checksum: 10c0/c39d6aa30e7a4e73965942ca5e13405ae05c9cb49f755a35f04248c864c0b24cf662d9767f1797b3ec48d1cf4e54774dce4a19c16534bd5cfd2aa3da81c9dc3a
+  checksum: 10c0/798e35d83bf48419a8c73de20bb94d68be5dde68de23f95d80a0ebe401e3b83e29e3e84aea7894d67fa6c79d2d3d40cc5bcde3e166f657ce50987aaa2421b6a9
   languageName: node
   linkType: hard
 
-"playwright@npm:1.55.0":
-  version: 1.55.0
-  resolution: "playwright@npm:1.55.0"
+"playwright@npm:1.57.0":
+  version: 1.57.0
+  resolution: "playwright@npm:1.57.0"
   dependencies:
     fsevents: "npm:2.3.2"
-    playwright-core: "npm:1.55.0"
+    playwright-core: "npm:1.57.0"
   dependenciesMeta:
     fsevents:
       optional: true
   bin:
     playwright: cli.js
-  checksum: 10c0/51605b7e57a5650e57972c5fdfc09d7a9934cca1cbee5beacca716fa801e25cb5bb7c1663de90c22b300fde884e5545a2b13a0505a93270b660687791c478304
+  checksum: 10c0/ab03c99a67b835bdea9059f516ad3b6e42c21025f9adaa161a4ef6bc7ca716dcba476d287140bb240d06126eb23f889a8933b8f5f1f1a56b80659d92d1358899
   languageName: node
   linkType: hard