chore(repo): e2e resiliency

.github/workflows/ci.yml

@@ -290,10 +290,95 @@ jobs:
           path: .turbo/runs
           retention-days: 5
 
-  integration-tests:
-    # needs: [check-permissions, build-packages]
+  integration-matrix:
     needs: [check-permissions]
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
+    name: Integration Matrix
+    runs-on: "blacksmith-8vcpu-ubuntu-2204"
+    defaults:
+      run:
+        shell: bash
+    timeout-minutes: ${{ vars.TIMEOUT_MINUTES_NORMAL && fromJSON(vars.TIMEOUT_MINUTES_NORMAL) || 10 }}
+
+    outputs:
+      has-tests: ${{ steps.compute.outputs.has-tests }}
+      matrix: ${{ steps.compute.outputs.matrix }}
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          fetch-tags: false
+          filter: "blob:none"
+          show-progress: false
+
+      - name: Setup
+        id: config
+        uses: ./.github/actions/init-blacksmith
+
+      - name: Compute affected integration matrix
+        id: compute
+        run: |
+          MATRIX_JSON="$(node scripts/ci/compute-integration-matrix.mjs)"
+          if [ "$MATRIX_JSON" = "[]" ]; then
+            echo "has-tests=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "has-tests=true" >> "$GITHUB_OUTPUT"
+          fi
+          echo "matrix=$MATRIX_JSON" >> "$GITHUB_OUTPUT"
+
+  integration-artifacts:
+    needs: [check-permissions, build-packages, integration-matrix]
+    if: ${{ needs.integration-matrix.outputs.has-tests == 'true' }}
+    name: Integration Artifacts
+    runs-on: "blacksmith-8vcpu-ubuntu-2204"
+    defaults:
+      run:
+        shell: bash
+    timeout-minutes: ${{ vars.TIMEOUT_MINUTES_LONG && fromJSON(vars.TIMEOUT_MINUTES_LONG) || 20 }}
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          fetch-tags: false
+          filter: "blob:none"
+          show-progress: false
+
+      - name: Setup
+        id: config
+        uses: ./.github/actions/init-blacksmith
+
+      - name: Version packages for snapshot
+        run: npm run version-packages:snapshot ci
+
+      - name: Build packages (default)
+        run: pnpm turbo build $TURBO_ARGS --only
+
+      - name: Pack @clerk/* tarballs (default)
+        run: pnpm pack -r --filter @clerk/* --pack-destination ./.integration-tarballs/default
+
+      - name: Build packages (RQ)
+        env:
+          CLERK_USE_RQ: true
+        run: pnpm turbo build $TURBO_ARGS --only --force
+
+      - name: Pack @clerk/* tarballs (RQ)
+        run: pnpm pack -r --filter @clerk/* --pack-destination ./.integration-tarballs/rq
+
+      - name: Upload integration tarballs
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-tarballs
+          if-no-files-found: error
+          path: ./.integration-tarballs
+          retention-days: 1
+
+  integration-tests:
+    needs: [check-permissions, integration-artifacts, integration-matrix]
+    if: ${{ (github.event_name != 'pull_request' || github.event.pull_request.draft == false) && needs.integration-matrix.outputs.has-tests == 'true' }}
     name: Integration Tests (${{ matrix.test-name }}, ${{ matrix.test-project }}${{ matrix.next-version && format(', {0}', matrix.next-version) || '' }}${{ matrix.clerk-use-rq == 'true' && ', RQ' || '' }})
     permissions:
       contents: read
@@ -307,53 +392,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test-name:
-          [
-            "generic",
-            "express",
-            "ap-flows",
-            "localhost",
-            "sessions",
-            "sessions:staging",
-            "handshake",
-            "handshake:staging",
-            "astro",
-            "tanstack-react-start",
-            "vue",
-            "nuxt",
-            "react-router",
-            "custom",
-          ]
-        test-project: ["chrome"]
-        include:
-          - test-name: "billing"
-            test-project: "chrome"
-            clerk-use-rq: "false"
-          - test-name: "billing"
-            test-project: "chrome"
-            clerk-use-rq: "true"
-          - test-name: "machine"
-            test-project: "chrome"
-            clerk-use-rq: "false"
-          - test-name: "machine"
-            test-project: "chrome"
-            clerk-use-rq: "true"
-          - test-name: "nextjs"
-            test-project: "chrome"
-            next-version: "15"
-          - test-name: "nextjs"
-            test-project: "chrome"
-            next-version: "16"
-            clerk-use-rq: "true"
-          - test-name: "nextjs"
-            test-project: "chrome"
-            next-version: "16"
-          - test-name: "quickstart"
-            test-project: "chrome"
-            next-version: "15"
-          - test-name: "quickstart"
-            test-project: "chrome"
-            next-version: "16"
+        include: ${{ fromJson(needs.integration-matrix.outputs.matrix) }}
 
     steps:
       - name: Checkout Repo
@@ -373,99 +412,55 @@ jobs:
           # turbo-token: ${{ secrets.TURBO_TOKEN }}
           playwright-enabled: true
 
-      - name: Verify jq is installed
-        shell: bash
-        run: |
-          if ! command -v jq &> /dev/null; then
-            echo "jq not found, installing..."
-            sudo apt-get update && sudo apt-get install -y jq
-          fi
-          jq --version
+      - name: Download integration tarballs
+        uses: actions/download-artifact@v4
+        with:
+          name: integration-tarballs
+          path: ./.integration-tarballs
 
-      - name: Task Status
-        id: task-status
-        env:
-          E2E_APP_CLERK_JS_DIR: ${{runner.temp}}
-          E2E_APP_CLERK_UI_DIR: ${{runner.temp}}
-          E2E_CLERK_JS_VERSION: "latest"
-          E2E_CLERK_UI_VERSION: "latest"
-          E2E_NEXTJS_VERSION: ${{ matrix.next-version }}
-          E2E_PROJECT: ${{ matrix.test-project }}
-          INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }}
+      - name: Select tarball variant
         run: |
-          # Use turbo's built-in --affected flag to detect changes
-          # This automatically uses GITHUB_BASE_REF in GitHub Actions
-          TASK_COUNT=$(pnpm turbo run test:integration:${{ matrix.test-name }} --affected --dry=json 2>/dev/null | jq '.tasks | length' 2>/dev/null || echo "0")
-
-          if [ "$TASK_COUNT" -gt 0 ]; then
-            AFFECTED=1
+          if [ "${{ matrix.clerk-use-rq }}" = "true" ]; then
+            echo "E2E_CLERK_TARBALLS_DIR=$GITHUB_WORKSPACE/.integration-tarballs/rq" >> "$GITHUB_ENV"
           else
-            AFFECTED=0
+            echo "E2E_CLERK_TARBALLS_DIR=$GITHUB_WORKSPACE/.integration-tarballs/default" >> "$GITHUB_ENV"
           fi
 
-          echo "affected=${AFFECTED}"
-          echo "affected=${AFFECTED}" >> $GITHUB_OUTPUT
-
-      - name: Rebuild @clerk/shared with CLERK_USE_RQ=true
-        if: ${{ steps.task-status.outputs.affected == '1' && matrix.clerk-use-rq == 'true' }}
-        run: pnpm turbo build $TURBO_ARGS --filter=@clerk/shared --force
-        env:
-          CLERK_USE_RQ: true
-
-      - name: Rebuild dependent packages with CLERK_USE_RQ=true
-        if: ${{ steps.task-status.outputs.affected == '1' && matrix.clerk-use-rq == 'true' }}
-        run: pnpm turbo build $TURBO_ARGS --filter=@clerk/shared^... --force
-        env:
-          CLERK_USE_RQ: true
-
-      - name: Version packages for snapshot
-        if: ${{ steps.task-status.outputs.affected == '1' }}
-        run: npm run version-packages:snapshot ci
-
-      - name: Verdaccio
-        if: ${{ steps.task-status.outputs.affected == '1' }}
-        uses: ./.github/actions/verdaccio
-        with:
-          publish-cmd: |
-            if [ "$(pnpm config get registry)" = "https://registry.npmjs.org/" ]; then echo 'Error: Using default registry' && exit 1; else ${{ matrix.clerk-use-rq == 'true' && 'CLERK_USE_RQ=true' || '' }} pnpm turbo build $TURBO_ARGS --only && pnpm changeset publish --no-git-tag --tag latest; fi
-
-      - name: Edit .npmrc [link-workspace-packages=false]
-        run: sed -i -E 's/link-workspace-packages=(deep|true)/link-workspace-packages=false/' .npmrc
+      - name: Resolve tarball paths
+        id: tarballs
+        run: |
+          echo "BACKEND_TGZ=$(ls "$E2E_CLERK_TARBALLS_DIR"/clerk-backend-*.tgz | head -1)" >> "$GITHUB_OUTPUT"
+          echo "CLERK_JS_TGZ=$(ls "$E2E_CLERK_TARBALLS_DIR"/clerk-clerk-js-*.tgz | head -1)" >> "$GITHUB_OUTPUT"
+          echo "UI_TGZ=$(ls "$E2E_CLERK_TARBALLS_DIR"/clerk-ui-*.tgz | head -1)" >> "$GITHUB_OUTPUT"
 
       - name: Install @clerk/backend in /integration
-        if: ${{ steps.task-status.outputs.affected == '1' }}
         working-directory: ./integration
         run: |
           pnpm init
           pnpm config set minimum-release-age-exclude @clerk/*
-          pnpm add @clerk/backend
+          pnpm add "${{ steps.tarballs.outputs.BACKEND_TGZ }}"
 
-      # Install published packages from Verdaccio to test against real npm install scenarios
-      # rather than local monorepo builds. Validates package structure, dependencies, and entry points.
       - name: Install @clerk/clerk-js in os temp
-        if: ${{ steps.task-status.outputs.affected == '1' }}
         working-directory: ${{runner.temp}}
         run: |
           mkdir clerk-js && cd clerk-js
           pnpm init
           pnpm config set minimum-release-age-exclude @clerk/*
-          pnpm add @clerk/clerk-js
+          pnpm add "${{ steps.tarballs.outputs.CLERK_JS_TGZ }}"
 
       - name: Install @clerk/ui in os temp
-        if: ${{ steps.task-status.outputs.affected == '1' }}
         working-directory: ${{runner.temp}}
         run: |
           mkdir clerk-ui && cd clerk-ui
           pnpm init
           pnpm config set minimum-release-age-exclude @clerk/*
-          pnpm add @clerk/ui
+          pnpm add "${{ steps.tarballs.outputs.UI_TGZ }}"
 
       - name: Copy components @clerk/astro
         if: ${{ matrix.test-name == 'astro' }}
         run: cd packages/astro && pnpm copy:components
 
       - name: Write all ENV certificates to files in integration/certs
-        if: ${{ steps.task-status.outputs.affected == '1' }}
         uses: actions/github-script@v7
         env:
           INTEGRATION_CERTS: "${{secrets.INTEGRATION_CERTS}}"
@@ -475,36 +470,32 @@ jobs:
             const fs = require('fs');
             const path = require('path');
             const rootCa = process.env.INTEGRATION_ROOT_CA;
-            console.log('rootCa', rootCa);
             fs.writeFileSync(path.join(process.env.GITHUB_WORKSPACE, 'integration/certs', 'rootCA.pem'), rootCa);
             const certs = JSON.parse(process.env.INTEGRATION_CERTS);
             for (const [name, cert] of Object.entries(certs)) {
                 fs.writeFileSync(path.join(process.env.GITHUB_WORKSPACE, 'integration/certs', name), cert);
             }
 
       - name: LS certs
-        if: ${{ steps.task-status.outputs.affected == '1' }}
         working-directory: ./integration/certs
         run: ls -la && pwd
 
       - name: Run Integration Tests
-        if: ${{ steps.task-status.outputs.affected == '1' }}
         id: integration-tests
         timeout-minutes: 25
         run: pnpm turbo test:integration:${{ matrix.test-name }} $TURBO_ARGS
         env:
-          E2E_DEBUG: "1"
+          E2E_DEBUG: ${{ github.run_attempt > 1 && '1' || '0' }}
           E2E_APP_CLERK_JS_DIR: ${{runner.temp}}
           E2E_APP_CLERK_UI_DIR: ${{runner.temp}}
-          E2E_CLERK_JS_VERSION: "latest"
-          E2E_CLERK_UI_VERSION: "latest"
           E2E_NEXTJS_VERSION: ${{ matrix.next-version }}
           E2E_PROJECT: ${{ matrix.test-project }}
           E2E_CLERK_ENCRYPTION_KEY: ${{ matrix.clerk-encryption-key }}
-          CLERK_USE_RQ: ${{ matrix.clerk-use-rq }}
+          CLERK_USE_RQ: ${{ matrix.clerk-use-rq == 'true' && 'true' || 'false' }}
           INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }}
           NODE_EXTRA_CA_CERTS: ${{ github.workspace }}/integration/certs/rootCA.pem
           VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
+          PLAYWRIGHT_WORKERS: "4"
 
       - name: Upload test-results
         if: ${{ cancelled() || failure() }}

integration/playwright.config.ts

@@ -14,7 +14,9 @@ export const common: PlaywrightTestConfig = {
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 5 : 0,
   maxFailures: process.env.CI ? 5 : undefined,
-  workers: process.env.CI ? '50%' : '70%',
+  workers: process.env.CI
+    ? (process.env.PLAYWRIGHT_WORKERS ? Number.parseInt(process.env.PLAYWRIGHT_WORKERS, 10) : '50%')
+    : '70%',
   use: {
     actionTimeout: 10_000,
     navigationTimeout: 30_000,

integration/presets/utils.ts

@@ -3,6 +3,21 @@ import path from 'node:path';
 export function linkPackage(pkg: string, tag?: string) {
   // eslint-disable-next-line turbo/no-undeclared-env-vars
   if (process.env.CI === 'true') {
+    const tarballsDir = process.env.E2E_CLERK_TARBALLS_DIR;
+    if (tarballsDir) {
+      // Resolve tarball by prefix so we don't need package.json versions to be snapshotted in every CI job.
+      // e.g. packages/shared -> clerk-shared-<version>.tgz
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      const fs = require('node:fs');
+      const prefix = `clerk-${pkg}-`;
+      const entries = fs.readdirSync(tarballsDir);
+      const match = entries.find((f: string) => f.startsWith(prefix) && f.endsWith('.tgz'));
+      if (!match) {
+        throw new Error(`Missing tarball for "${pkg}" in ${tarballsDir}`);
+      }
+      return `file:${path.resolve(tarballsDir, match)}`;
+    }
+
     // In CI, use '*' to get the latest version from Verdaccio
     // which will be the snapshot version we just published
     return '*';

integration/tests/session-tasks-multi-session.test.ts

@@ -81,7 +81,8 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withSessionTasks] })(
       // Navigate to protected page, with active session, where user button gets rendered
       await u.page.goToRelative('/user-button');
 
-      // Switch account, to a session that has a pending status
+      // Wait for page to be ready and user button to be mounted
+      await page.waitForLoadState('networkidle');
       await u.po.userButton.waitForMounted();
       await u.po.userButton.toggleTrigger();
       await u.po.userButton.waitForPopover();

integration/tests/session-tasks-sign-in-reset-password.test.ts

@@ -28,12 +28,16 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withSessionTasksResetPassword
       await u.po.signIn.setPassword(user.password);
       await u.po.signIn.continue();
 
+      // Wait for alternative methods to be visible, then check for error message
+      const altMethodsButton = u.po.signIn.getAltMethodsEmailCodeButton();
+      await expect(altMethodsButton).toBeVisible({ timeout: 10000 });
+
       await expect(
         u.page.getByText(
           'Your password may be compromised. To protect your account, please continue with an alternative sign-in method. You will be required to reset your password after signing in.',
         ),
-      ).toBeVisible();
-      await u.po.signIn.getAltMethodsEmailCodeButton().click();
+      ).toBeVisible({ timeout: 10000 });
+      await altMethodsButton.click();
 
       await u.page.getByRole('textbox', { name: 'code' }).click();
       await u.page.keyboard.type('424242', { delay: 100 });
@@ -81,12 +85,16 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withSessionTasksResetPassword
       await u.po.signIn.setPassword(user.password);
       await u.po.signIn.continue();
 
+      // Wait for alternative methods to be visible, then check for error message
+      const altMethodsButton = u.po.signIn.getAltMethodsEmailCodeButton();
+      await expect(altMethodsButton).toBeVisible({ timeout: 10000 });
+
       await expect(
         u.page.getByText(
           'Your password may be compromised. To protect your account, please continue with an alternative sign-in method. You will be required to reset your password after signing in.',
         ),
-      ).toBeVisible();
-      await u.po.signIn.getAltMethodsEmailCodeButton().click();
+      ).toBeVisible({ timeout: 10000 });
+      await altMethodsButton.click();
 
       await u.page.getByRole('textbox', { name: 'code' }).click();
       await u.page.keyboard.type('424242', { delay: 100 });