From 5d33f60e2cd39aa89ead04f965e79679b11f0453 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Sun, 1 Mar 2026 19:29:16 +0100 Subject: [PATCH 1/2] feat: add --spec-version option to tier-check --- .../checks/test-conformance-results.ts | 76 ++++++++++++------- src/tier-check/index.ts | 22 +++++- 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/src/tier-check/checks/test-conformance-results.ts b/src/tier-check/checks/test-conformance-results.ts index f68d879..2a9ce43 100644 --- a/src/tier-check/checks/test-conformance-results.ts +++ b/src/tier-check/checks/test-conformance-results.ts @@ -6,6 +6,8 @@ import { ConformanceResult } from '../types'; import { listScenarios, listActiveClientScenarios, + listScenariosForSpec, + listClientScenariosForSpec, getScenarioSpecVersions } from '../../scenarios'; import { ConformanceCheck, SpecVersion } from '../../types'; @@ -166,6 +168,7 @@ function reconcileWithExpected( export async function checkConformance(options: { serverUrl?: string; skip?: boolean; + specVersion?: SpecVersion; }): Promise { if (options.skip || !options.serverUrl) { return { @@ -179,23 +182,37 @@ export async function checkConformance(options: { } const outputDir = mkdtempSync(join(tmpdir(), 'tier-check-server-')); + const args = [ + process.argv[1], + 'server', + '--url', + options.serverUrl, + '-o', + outputDir + ]; + if (options.specVersion) { + args.push('--spec-version', options.specVersion); + } try { - execFileSync( - process.execPath, - [process.argv[1], 'server', '--url', options.serverUrl, '-o', outputDir], - { - stdio: ['pipe', 'pipe', 'pipe'], - timeout: 120_000 - } - ); + execFileSync(process.execPath, args, { + stdio: ['pipe', 'pipe', 'pipe'], + timeout: 120_000 + }); } catch { // Non-zero exit is expected when tests fail — results are still in outputDir } + const activeScenarios = new Set(listActiveClientScenarios()); + const expectedScenarios = options.specVersion + ? listClientScenariosForSpec(options.specVersion).filter((s) => + activeScenarios.has(s) + ) + : [...activeScenarios]; + return reconcileWithExpected( parseOutputDir(outputDir), - listActiveClientScenarios(), + expectedScenarios, 'server' ); } @@ -206,6 +223,7 @@ export async function checkConformance(options: { export async function checkClientConformance(options: { clientCmd?: string; skip?: boolean; + specVersion?: SpecVersion; }): Promise { if (options.skip || !options.clientCmd) { return { @@ -219,28 +237,32 @@ export async function checkClientConformance(options: { } const outputDir = mkdtempSync(join(tmpdir(), 'tier-check-client-')); + const args = [ + process.argv[1], + 'client', + '--command', + options.clientCmd, + '--suite', + 'all', + '-o', + outputDir + ]; + if (options.specVersion) { + args.push('--spec-version', options.specVersion); + } try { - execFileSync( - process.execPath, - [ - process.argv[1], - 'client', - '--command', - options.clientCmd, - '--suite', - 'all', - '-o', - outputDir - ], - { - stdio: ['pipe', 'pipe', 'pipe'], - timeout: 120_000 - } - ); + execFileSync(process.execPath, args, { + stdio: ['pipe', 'pipe', 'pipe'], + timeout: 120_000 + }); } catch { // Non-zero exit is expected when tests fail — results are still in outputDir } - return reconcileWithExpected(parseOutputDir(outputDir), listScenarios()); + const expectedScenarios = options.specVersion + ? listScenariosForSpec(options.specVersion) + : listScenarios(); + + return reconcileWithExpected(parseOutputDir(outputDir), expectedScenarios); } diff --git a/src/tier-check/index.ts b/src/tier-check/index.ts index 5416748..374286c 100644 --- a/src/tier-check/index.ts +++ b/src/tier-check/index.ts @@ -13,6 +13,8 @@ import { checkSpecTracking } from './checks/spec-tracking'; import { computeTier } from './tier-logic'; import { formatJson, formatMarkdown, formatTerminal } from './output'; import { TierScorecard } from './types'; +import { ALL_SPEC_VERSIONS } from '../scenarios'; +import { SpecVersion } from '../types'; function parseRepo(repo: string): { owner: string; repo: string } { const parts = repo.split('/'); @@ -48,10 +50,24 @@ export function createTierCheckCommand(): Command { '--token ', 'GitHub token (defaults to GITHUB_TOKEN env var)' ) + .option( + '--spec-version ', + 'Only run conformance scenarios for this spec version' + ) .action(async (options) => { const { owner, repo } = parseRepo(options.repo); let token = options.token || process.env.GITHUB_TOKEN; + let specVersion: SpecVersion | undefined; + if (options.specVersion) { + if (!ALL_SPEC_VERSIONS.includes(options.specVersion as SpecVersion)) { + console.error(`Unknown spec version: ${options.specVersion}`); + console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`); + process.exit(1); + } + specVersion = options.specVersion as SpecVersion; + } + if (!token) { // Try to get token from GitHub CLI try { @@ -90,14 +106,16 @@ export function createTierCheckCommand(): Command { ] = await Promise.all([ checkConformance({ serverUrl: options.conformanceServerUrl, - skip: options.skipConformance + skip: options.skipConformance, + specVersion }).then((r) => { console.error(' ✓ Server Conformance'); return r; }), checkClientConformance({ clientCmd: options.clientCmd, - skip: options.skipConformance || !options.clientCmd + skip: options.skipConformance || !options.clientCmd, + specVersion }).then((r) => { console.error(' ✓ Client Conformance'); return r; From 6a68603e43fe6267b4531b21e7f3f02b865a905f Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Wed, 25 Mar 2026 14:46:56 +0000 Subject: [PATCH 2/2] refactor: share resolveSpecVersion between CLI and tier-check Move resolveSpecVersion to src/scenarios/index.ts so tier-check can reuse it instead of duplicating the validation logic. --- src/index.ts | 11 +---------- src/scenarios/index.ts | 9 +++++++++ src/tier-check/index.ts | 15 ++++----------- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/index.ts b/src/index.ts index 9aeb0f0..29edd4a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,7 +24,7 @@ import { listScenariosForSpec, listClientScenariosForSpec, getScenarioSpecVersions, - ALL_SPEC_VERSIONS + resolveSpecVersion } from './scenarios'; import type { SpecVersion } from './scenarios'; import { ConformanceCheck } from './types'; @@ -37,15 +37,6 @@ import { import { createTierCheckCommand } from './tier-check'; import packageJson from '../package.json'; -function resolveSpecVersion(value: string): SpecVersion { - if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) { - return value as SpecVersion; - } - console.error(`Unknown spec version: ${value}`); - console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`); - process.exit(1); -} - // Note on naming: `command` refers to which CLI command is calling this. // The `client` command tests Scenario objects (which test clients), // and the `server` command tests ClientScenario objects (which test servers). diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 36c8f32..75ca61d 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -227,6 +227,15 @@ export const ALL_SPEC_VERSIONS: SpecVersion[] = [ 'extension' ]; +export function resolveSpecVersion(value: string): SpecVersion { + if (ALL_SPEC_VERSIONS.includes(value as SpecVersion)) { + return value as SpecVersion; + } + console.error(`Unknown spec version: ${value}`); + console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`); + process.exit(1); +} + export function listScenariosForSpec(version: SpecVersion): string[] { return scenariosList .filter((s) => s.specVersions.includes(version)) diff --git a/src/tier-check/index.ts b/src/tier-check/index.ts index 374286c..1cd32c1 100644 --- a/src/tier-check/index.ts +++ b/src/tier-check/index.ts @@ -13,8 +13,7 @@ import { checkSpecTracking } from './checks/spec-tracking'; import { computeTier } from './tier-logic'; import { formatJson, formatMarkdown, formatTerminal } from './output'; import { TierScorecard } from './types'; -import { ALL_SPEC_VERSIONS } from '../scenarios'; -import { SpecVersion } from '../types'; +import { resolveSpecVersion } from '../scenarios'; function parseRepo(repo: string): { owner: string; repo: string } { const parts = repo.split('/'); @@ -58,15 +57,9 @@ export function createTierCheckCommand(): Command { const { owner, repo } = parseRepo(options.repo); let token = options.token || process.env.GITHUB_TOKEN; - let specVersion: SpecVersion | undefined; - if (options.specVersion) { - if (!ALL_SPEC_VERSIONS.includes(options.specVersion as SpecVersion)) { - console.error(`Unknown spec version: ${options.specVersion}`); - console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`); - process.exit(1); - } - specVersion = options.specVersion as SpecVersion; - } + const specVersion = options.specVersion + ? resolveSpecVersion(options.specVersion) + : undefined; if (!token) { // Try to get token from GitHub CLI