diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b10b6438a55..d47695dd812 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -290,10 +290,95 @@ jobs: path: .turbo/runs retention-days: 5 - integration-tests: - # needs: [check-permissions, build-packages] + integration-matrix: needs: [check-permissions] if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} + name: Integration Matrix + runs-on: "blacksmith-8vcpu-ubuntu-2204" + defaults: + run: + shell: bash + timeout-minutes: ${{ vars.TIMEOUT_MINUTES_NORMAL && fromJSON(vars.TIMEOUT_MINUTES_NORMAL) || 10 }} + + outputs: + has-tests: ${{ steps.compute.outputs.has-tests }} + matrix: ${{ steps.compute.outputs.matrix }} + + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + fetch-depth: 1 + fetch-tags: false + filter: "blob:none" + show-progress: false + + - name: Setup + id: config + uses: ./.github/actions/init-blacksmith + + - name: Compute affected integration matrix + id: compute + run: | + MATRIX_JSON="$(node scripts/ci/compute-integration-matrix.mjs)" + if [ "$MATRIX_JSON" = "[]" ]; then + echo "has-tests=false" >> "$GITHUB_OUTPUT" + else + echo "has-tests=true" >> "$GITHUB_OUTPUT" + fi + echo "matrix=$MATRIX_JSON" >> "$GITHUB_OUTPUT" + + integration-artifacts: + needs: [check-permissions, build-packages, integration-matrix] + if: ${{ needs.integration-matrix.outputs.has-tests == 'true' }} + name: Integration Artifacts + runs-on: "blacksmith-8vcpu-ubuntu-2204" + defaults: + run: + shell: bash + timeout-minutes: ${{ vars.TIMEOUT_MINUTES_LONG && fromJSON(vars.TIMEOUT_MINUTES_LONG) || 20 }} + + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + with: + fetch-depth: 1 + fetch-tags: false + filter: "blob:none" + show-progress: false + + - name: Setup + id: config + uses: ./.github/actions/init-blacksmith + + - name: Version packages for snapshot + run: npm run version-packages:snapshot ci + + - name: Build packages (default) + run: pnpm turbo build $TURBO_ARGS --only + + - name: Pack @clerk/* tarballs (default) + run: pnpm pack -r --filter @clerk/* --pack-destination ./.integration-tarballs/default + + - name: Build packages (RQ) + env: + CLERK_USE_RQ: true + run: pnpm turbo build $TURBO_ARGS --only --force + + - name: Pack @clerk/* tarballs (RQ) + run: pnpm pack -r --filter @clerk/* --pack-destination ./.integration-tarballs/rq + + - name: Upload integration tarballs + uses: actions/upload-artifact@v4 + with: + name: integration-tarballs + if-no-files-found: error + path: ./.integration-tarballs + retention-days: 1 + + integration-tests: + needs: [check-permissions, integration-artifacts, integration-matrix] + if: ${{ (github.event_name != 'pull_request' || github.event.pull_request.draft == false) && needs.integration-matrix.outputs.has-tests == 'true' }} name: Integration Tests (${{ matrix.test-name }}, ${{ matrix.test-project }}${{ matrix.next-version && format(', {0}', matrix.next-version) || '' }}${{ matrix.clerk-use-rq == 'true' && ', RQ' || '' }}) permissions: contents: read @@ -307,53 +392,7 @@ jobs: strategy: fail-fast: false matrix: - test-name: - [ - "generic", - "express", - "ap-flows", - "localhost", - "sessions", - "sessions:staging", - "handshake", - "handshake:staging", - "astro", - "tanstack-react-start", - "vue", - "nuxt", - "react-router", - "custom", - ] - test-project: ["chrome"] - include: - - test-name: "billing" - test-project: "chrome" - clerk-use-rq: "false" - - test-name: "billing" - test-project: "chrome" - clerk-use-rq: "true" - - test-name: "machine" - test-project: "chrome" - clerk-use-rq: "false" - - test-name: "machine" - test-project: "chrome" - clerk-use-rq: "true" - - test-name: "nextjs" - test-project: "chrome" - next-version: "15" - - test-name: "nextjs" - test-project: "chrome" - next-version: "16" - clerk-use-rq: "true" - - test-name: "nextjs" - test-project: "chrome" - next-version: "16" - - test-name: "quickstart" - test-project: "chrome" - next-version: "15" - - test-name: "quickstart" - test-project: "chrome" - next-version: "16" + include: ${{ fromJson(needs.integration-matrix.outputs.matrix) }} steps: - name: Checkout Repo @@ -373,99 +412,55 @@ jobs: # turbo-token: ${{ secrets.TURBO_TOKEN }} playwright-enabled: true - - name: Verify jq is installed - shell: bash - run: | - if ! command -v jq &> /dev/null; then - echo "jq not found, installing..." - sudo apt-get update && sudo apt-get install -y jq - fi - jq --version + - name: Download integration tarballs + uses: actions/download-artifact@v4 + with: + name: integration-tarballs + path: ./.integration-tarballs - - name: Task Status - id: task-status - env: - E2E_APP_CLERK_JS_DIR: ${{runner.temp}} - E2E_APP_CLERK_UI_DIR: ${{runner.temp}} - E2E_CLERK_JS_VERSION: "latest" - E2E_CLERK_UI_VERSION: "latest" - E2E_NEXTJS_VERSION: ${{ matrix.next-version }} - E2E_PROJECT: ${{ matrix.test-project }} - INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }} + - name: Select tarball variant run: | - # Use turbo's built-in --affected flag to detect changes - # This automatically uses GITHUB_BASE_REF in GitHub Actions - TASK_COUNT=$(pnpm turbo run test:integration:${{ matrix.test-name }} --affected --dry=json 2>/dev/null | jq '.tasks | length' 2>/dev/null || echo "0") - - if [ "$TASK_COUNT" -gt 0 ]; then - AFFECTED=1 + if [ "${{ matrix.clerk-use-rq }}" = "true" ]; then + echo "E2E_CLERK_TARBALLS_DIR=$GITHUB_WORKSPACE/.integration-tarballs/rq" >> "$GITHUB_ENV" else - AFFECTED=0 + echo "E2E_CLERK_TARBALLS_DIR=$GITHUB_WORKSPACE/.integration-tarballs/default" >> "$GITHUB_ENV" fi - echo "affected=${AFFECTED}" - echo "affected=${AFFECTED}" >> $GITHUB_OUTPUT - - - name: Rebuild @clerk/shared with CLERK_USE_RQ=true - if: ${{ steps.task-status.outputs.affected == '1' && matrix.clerk-use-rq == 'true' }} - run: pnpm turbo build $TURBO_ARGS --filter=@clerk/shared --force - env: - CLERK_USE_RQ: true - - - name: Rebuild dependent packages with CLERK_USE_RQ=true - if: ${{ steps.task-status.outputs.affected == '1' && matrix.clerk-use-rq == 'true' }} - run: pnpm turbo build $TURBO_ARGS --filter=@clerk/shared^... --force - env: - CLERK_USE_RQ: true - - - name: Version packages for snapshot - if: ${{ steps.task-status.outputs.affected == '1' }} - run: npm run version-packages:snapshot ci - - - name: Verdaccio - if: ${{ steps.task-status.outputs.affected == '1' }} - uses: ./.github/actions/verdaccio - with: - publish-cmd: | - if [ "$(pnpm config get registry)" = "https://registry.npmjs.org/" ]; then echo 'Error: Using default registry' && exit 1; else ${{ matrix.clerk-use-rq == 'true' && 'CLERK_USE_RQ=true' || '' }} pnpm turbo build $TURBO_ARGS --only && pnpm changeset publish --no-git-tag --tag latest; fi - - - name: Edit .npmrc [link-workspace-packages=false] - run: sed -i -E 's/link-workspace-packages=(deep|true)/link-workspace-packages=false/' .npmrc + - name: Resolve tarball paths + id: tarballs + run: | + echo "BACKEND_TGZ=$(ls "$E2E_CLERK_TARBALLS_DIR"/clerk-backend-*.tgz | head -1)" >> "$GITHUB_OUTPUT" + echo "CLERK_JS_TGZ=$(ls "$E2E_CLERK_TARBALLS_DIR"/clerk-clerk-js-*.tgz | head -1)" >> "$GITHUB_OUTPUT" + echo "UI_TGZ=$(ls "$E2E_CLERK_TARBALLS_DIR"/clerk-ui-*.tgz | head -1)" >> "$GITHUB_OUTPUT" - name: Install @clerk/backend in /integration - if: ${{ steps.task-status.outputs.affected == '1' }} working-directory: ./integration run: | pnpm init pnpm config set minimum-release-age-exclude @clerk/* - pnpm add @clerk/backend + pnpm add "${{ steps.tarballs.outputs.BACKEND_TGZ }}" - # Install published packages from Verdaccio to test against real npm install scenarios - # rather than local monorepo builds. Validates package structure, dependencies, and entry points. - name: Install @clerk/clerk-js in os temp - if: ${{ steps.task-status.outputs.affected == '1' }} working-directory: ${{runner.temp}} run: | mkdir clerk-js && cd clerk-js pnpm init pnpm config set minimum-release-age-exclude @clerk/* - pnpm add @clerk/clerk-js + pnpm add "${{ steps.tarballs.outputs.CLERK_JS_TGZ }}" - name: Install @clerk/ui in os temp - if: ${{ steps.task-status.outputs.affected == '1' }} working-directory: ${{runner.temp}} run: | mkdir clerk-ui && cd clerk-ui pnpm init pnpm config set minimum-release-age-exclude @clerk/* - pnpm add @clerk/ui + pnpm add "${{ steps.tarballs.outputs.UI_TGZ }}" - name: Copy components @clerk/astro if: ${{ matrix.test-name == 'astro' }} run: cd packages/astro && pnpm copy:components - name: Write all ENV certificates to files in integration/certs - if: ${{ steps.task-status.outputs.affected == '1' }} uses: actions/github-script@v7 env: INTEGRATION_CERTS: "${{secrets.INTEGRATION_CERTS}}" @@ -475,7 +470,6 @@ jobs: const fs = require('fs'); const path = require('path'); const rootCa = process.env.INTEGRATION_ROOT_CA; - console.log('rootCa', rootCa); fs.writeFileSync(path.join(process.env.GITHUB_WORKSPACE, 'integration/certs', 'rootCA.pem'), rootCa); const certs = JSON.parse(process.env.INTEGRATION_CERTS); for (const [name, cert] of Object.entries(certs)) { @@ -483,28 +477,25 @@ jobs: } - name: LS certs - if: ${{ steps.task-status.outputs.affected == '1' }} working-directory: ./integration/certs run: ls -la && pwd - name: Run Integration Tests - if: ${{ steps.task-status.outputs.affected == '1' }} id: integration-tests timeout-minutes: 25 run: pnpm turbo test:integration:${{ matrix.test-name }} $TURBO_ARGS env: - E2E_DEBUG: "1" + E2E_DEBUG: ${{ github.run_attempt > 1 && '1' || '0' }} E2E_APP_CLERK_JS_DIR: ${{runner.temp}} E2E_APP_CLERK_UI_DIR: ${{runner.temp}} - E2E_CLERK_JS_VERSION: "latest" - E2E_CLERK_UI_VERSION: "latest" E2E_NEXTJS_VERSION: ${{ matrix.next-version }} E2E_PROJECT: ${{ matrix.test-project }} E2E_CLERK_ENCRYPTION_KEY: ${{ matrix.clerk-encryption-key }} - CLERK_USE_RQ: ${{ matrix.clerk-use-rq }} + CLERK_USE_RQ: ${{ matrix.clerk-use-rq == 'true' && 'true' || 'false' }} INTEGRATION_INSTANCE_KEYS: ${{ secrets.INTEGRATION_INSTANCE_KEYS }} NODE_EXTRA_CA_CERTS: ${{ github.workspace }}/integration/certs/rootCA.pem VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + PLAYWRIGHT_WORKERS: "4" - name: Upload test-results if: ${{ cancelled() || failure() }} diff --git a/integration/playwright.config.ts b/integration/playwright.config.ts index 007d17768cc..5163b801ff6 100644 --- a/integration/playwright.config.ts +++ b/integration/playwright.config.ts @@ -14,7 +14,9 @@ export const common: PlaywrightTestConfig = { forbidOnly: !!process.env.CI, retries: process.env.CI ? 5 : 0, maxFailures: process.env.CI ? 5 : undefined, - workers: process.env.CI ? '50%' : '70%', + workers: process.env.CI + ? (process.env.PLAYWRIGHT_WORKERS ? Number.parseInt(process.env.PLAYWRIGHT_WORKERS, 10) : '50%') + : '70%', use: { actionTimeout: 10_000, navigationTimeout: 30_000, diff --git a/integration/presets/utils.ts b/integration/presets/utils.ts index 21672d16b5b..1fb1f2e9c68 100644 --- a/integration/presets/utils.ts +++ b/integration/presets/utils.ts @@ -3,6 +3,21 @@ import path from 'node:path'; export function linkPackage(pkg: string, tag?: string) { // eslint-disable-next-line turbo/no-undeclared-env-vars if (process.env.CI === 'true') { + const tarballsDir = process.env.E2E_CLERK_TARBALLS_DIR; + if (tarballsDir) { + // Resolve tarball by prefix so we don't need package.json versions to be snapshotted in every CI job. + // e.g. packages/shared -> clerk-shared-.tgz + // eslint-disable-next-line @typescript-eslint/no-require-imports + const fs = require('node:fs'); + const prefix = `clerk-${pkg}-`; + const entries = fs.readdirSync(tarballsDir); + const match = entries.find((f: string) => f.startsWith(prefix) && f.endsWith('.tgz')); + if (!match) { + throw new Error(`Missing tarball for "${pkg}" in ${tarballsDir}`); + } + return `file:${path.resolve(tarballsDir, match)}`; + } + // In CI, use '*' to get the latest version from Verdaccio // which will be the snapshot version we just published return '*'; diff --git a/integration/tests/session-tasks-multi-session.test.ts b/integration/tests/session-tasks-multi-session.test.ts index da265d1f693..f20c88a24f8 100644 --- a/integration/tests/session-tasks-multi-session.test.ts +++ b/integration/tests/session-tasks-multi-session.test.ts @@ -81,7 +81,8 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withSessionTasks] })( // Navigate to protected page, with active session, where user button gets rendered await u.page.goToRelative('/user-button'); - // Switch account, to a session that has a pending status + // Wait for page to be ready and user button to be mounted + await page.waitForLoadState('networkidle'); await u.po.userButton.waitForMounted(); await u.po.userButton.toggleTrigger(); await u.po.userButton.waitForPopover(); diff --git a/integration/tests/session-tasks-sign-in-reset-password.test.ts b/integration/tests/session-tasks-sign-in-reset-password.test.ts index ac303f19473..6da64429639 100644 --- a/integration/tests/session-tasks-sign-in-reset-password.test.ts +++ b/integration/tests/session-tasks-sign-in-reset-password.test.ts @@ -28,12 +28,16 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withSessionTasksResetPassword await u.po.signIn.setPassword(user.password); await u.po.signIn.continue(); + // Wait for alternative methods to be visible, then check for error message + const altMethodsButton = u.po.signIn.getAltMethodsEmailCodeButton(); + await expect(altMethodsButton).toBeVisible({ timeout: 10000 }); + await expect( u.page.getByText( 'Your password may be compromised. To protect your account, please continue with an alternative sign-in method. You will be required to reset your password after signing in.', ), - ).toBeVisible(); - await u.po.signIn.getAltMethodsEmailCodeButton().click(); + ).toBeVisible({ timeout: 10000 }); + await altMethodsButton.click(); await u.page.getByRole('textbox', { name: 'code' }).click(); await u.page.keyboard.type('424242', { delay: 100 }); @@ -81,12 +85,16 @@ testAgainstRunningApps({ withEnv: [appConfigs.envs.withSessionTasksResetPassword await u.po.signIn.setPassword(user.password); await u.po.signIn.continue(); + // Wait for alternative methods to be visible, then check for error message + const altMethodsButton = u.po.signIn.getAltMethodsEmailCodeButton(); + await expect(altMethodsButton).toBeVisible({ timeout: 10000 }); + await expect( u.page.getByText( 'Your password may be compromised. To protect your account, please continue with an alternative sign-in method. You will be required to reset your password after signing in.', ), - ).toBeVisible(); - await u.po.signIn.getAltMethodsEmailCodeButton().click(); + ).toBeVisible({ timeout: 10000 }); + await altMethodsButton.click(); await u.page.getByRole('textbox', { name: 'code' }).click(); await u.page.keyboard.type('424242', { delay: 100 }); diff --git a/packages/testing/src/playwright/unstable/page-objects/sessionTask.ts b/packages/testing/src/playwright/unstable/page-objects/sessionTask.ts index 3a38d34e062..17d42efd043 100644 --- a/packages/testing/src/playwright/unstable/page-objects/sessionTask.ts +++ b/packages/testing/src/playwright/unstable/page-objects/sessionTask.ts @@ -27,8 +27,14 @@ export const createSessionTaskComponentPageObject = (testArgs: { page: EnhancedP newPassword: string; confirmPassword: string; }) => { - await page.locator('input[name=newPassword]').fill(newPassword); - await page.locator('input[name=confirmPassword]').fill(confirmPassword); + const newPasswordInput = page.locator('input[name=newPassword]'); + const confirmPasswordInput = page.locator('input[name=confirmPassword]'); + + await expect(newPasswordInput).toBeVisible(); + await expect(confirmPasswordInput).toBeVisible(); + + await newPasswordInput.fill(newPassword); + await confirmPasswordInput.fill(confirmPassword); const resetPasswordButton = page.getByRole('button', { name: /reset password/i }); await expect(resetPasswordButton).toBeVisible(); diff --git a/scripts/ci/compute-integration-matrix.mjs b/scripts/ci/compute-integration-matrix.mjs new file mode 100644 index 00000000000..bc484006e3a --- /dev/null +++ b/scripts/ci/compute-integration-matrix.mjs @@ -0,0 +1,67 @@ +import { spawnSync } from 'node:child_process'; + +/** + * Outputs a JSON array usable as a GitHub Actions matrix `include`. + * Filters to only suites where Turborepo reports affected tasks. + */ +function main() { + const allEntries = [ + { 'test-name': 'generic', 'test-project': 'chrome' }, + { 'test-name': 'express', 'test-project': 'chrome' }, + { 'test-name': 'ap-flows', 'test-project': 'chrome' }, + { 'test-name': 'localhost', 'test-project': 'chrome' }, + { 'test-name': 'sessions', 'test-project': 'chrome' }, + { 'test-name': 'sessions:staging', 'test-project': 'chrome' }, + { 'test-name': 'handshake', 'test-project': 'chrome' }, + { 'test-name': 'handshake:staging', 'test-project': 'chrome' }, + { 'test-name': 'astro', 'test-project': 'chrome' }, + { 'test-name': 'tanstack-react-start', 'test-project': 'chrome' }, + { 'test-name': 'vue', 'test-project': 'chrome' }, + { 'test-name': 'nuxt', 'test-project': 'chrome' }, + { 'test-name': 'react-router', 'test-project': 'chrome' }, + { 'test-name': 'custom', 'test-project': 'chrome' }, + + { 'clerk-use-rq': 'false', 'test-name': 'billing', 'test-project': 'chrome' }, + { 'clerk-use-rq': 'true', 'test-name': 'billing', 'test-project': 'chrome' }, + { 'clerk-use-rq': 'false', 'test-name': 'machine', 'test-project': 'chrome' }, + { 'clerk-use-rq': 'true', 'test-name': 'machine', 'test-project': 'chrome' }, + + { 'next-version': '15', 'test-name': 'nextjs', 'test-project': 'chrome' }, + { 'clerk-use-rq': 'true', 'next-version': '16', 'test-name': 'nextjs', 'test-project': 'chrome' }, + { 'next-version': '16', 'test-name': 'nextjs', 'test-project': 'chrome' }, + + { 'next-version': '15', 'test-name': 'quickstart', 'test-project': 'chrome' }, + { 'next-version': '16', 'test-name': 'quickstart', 'test-project': 'chrome' }, + ]; + + const turboArgs = (process.env.TURBO_ARGS ?? '').split(' ').filter(Boolean); + const uniqueSuites = [...new Set(allEntries.map(e => e['test-name']))]; + + const affectedSuites = new Set( + uniqueSuites.filter(suite => { + const result = spawnSync( + 'pnpm', + ['turbo', 'run', `test:integration:${suite}`, '--affected', '--dry=json', ...turboArgs], + { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }, + ); + + if (result.status !== 0 || !result.stdout) { + return false; + } + + try { + const parsed = JSON.parse(result.stdout); + return Array.isArray(parsed?.tasks) && parsed.tasks.length > 0; + } catch { + return false; + } + }), + ); + + const matrix = allEntries.filter(e => affectedSuites.has(e['test-name'])); + process.stdout.write(JSON.stringify(matrix)); +} + +main(); + +