diff --git a/src/tier-check/checks/test-conformance-results.ts b/src/tier-check/checks/test-conformance-results.ts index 5ba9849..0c6a007 100644 --- a/src/tier-check/checks/test-conformance-results.ts +++ b/src/tier-check/checks/test-conformance-results.ts @@ -6,6 +6,8 @@ import { ConformanceResult } from '../types'; import { listScenarios, listActiveClientScenarios, + listScenariosForSpec, + listClientScenariosForSpec, getScenarioSpecVersions } from '../../scenarios'; import { ConformanceCheck, SpecVersion } from '../../types'; @@ -166,6 +168,7 @@ function reconcileWithExpected( export async function checkConformance(options: { serverUrl?: string; skip?: boolean; + specVersion?: SpecVersion; }): Promise { if (options.skip || !options.serverUrl) { return { @@ -179,10 +182,13 @@ export async function checkConformance(options: { } const outputDir = mkdtempSync(join(tmpdir(), 'tier-check-server-')); + const specVersionArg = options.specVersion + ? `--spec-version ${options.specVersion}` + : ''; try { execSync( - `node dist/index.js server --url ${options.serverUrl} -o ${outputDir}`, + `node dist/index.js server --url ${options.serverUrl} -o ${outputDir} ${specVersionArg}`, { cwd: process.cwd(), stdio: ['pipe', 'pipe', 'pipe'], @@ -193,9 +199,16 @@ export async function checkConformance(options: { // Non-zero exit is expected when tests fail — results are still in outputDir } + const activeScenarios = new Set(listActiveClientScenarios()); + const expectedScenarios = options.specVersion + ? listClientScenariosForSpec(options.specVersion).filter((s) => + activeScenarios.has(s) + ) + : [...activeScenarios]; + return reconcileWithExpected( parseOutputDir(outputDir), - listActiveClientScenarios(), + expectedScenarios, 'server' ); } @@ -206,6 +219,7 @@ export async function checkConformance(options: { export async function checkClientConformance(options: { clientCmd?: string; skip?: boolean; + specVersion?: SpecVersion; }): Promise { if (options.skip || !options.clientCmd) { return { @@ -219,10 +233,13 @@ export async function checkClientConformance(options: { } const outputDir = mkdtempSync(join(tmpdir(), 'tier-check-client-')); + const specVersionArg = options.specVersion + ? `--spec-version ${options.specVersion}` + : ''; try { execSync( - `node dist/index.js client --command '${options.clientCmd}' --suite all -o ${outputDir}`, + `node dist/index.js client --command '${options.clientCmd}' --suite all -o ${outputDir} ${specVersionArg}`, { cwd: process.cwd(), stdio: ['pipe', 'pipe', 'pipe'], @@ -233,5 +250,9 @@ export async function checkClientConformance(options: { // Non-zero exit is expected when tests fail — results are still in outputDir } - return reconcileWithExpected(parseOutputDir(outputDir), listScenarios()); + const expectedScenarios = options.specVersion + ? listScenariosForSpec(options.specVersion) + : listScenarios(); + + return reconcileWithExpected(parseOutputDir(outputDir), expectedScenarios); } diff --git a/src/tier-check/index.ts b/src/tier-check/index.ts index 5416748..374286c 100644 --- a/src/tier-check/index.ts +++ b/src/tier-check/index.ts @@ -13,6 +13,8 @@ import { checkSpecTracking } from './checks/spec-tracking'; import { computeTier } from './tier-logic'; import { formatJson, formatMarkdown, formatTerminal } from './output'; import { TierScorecard } from './types'; +import { ALL_SPEC_VERSIONS } from '../scenarios'; +import { SpecVersion } from '../types'; function parseRepo(repo: string): { owner: string; repo: string } { const parts = repo.split('/'); @@ -48,10 +50,24 @@ export function createTierCheckCommand(): Command { '--token ', 'GitHub token (defaults to GITHUB_TOKEN env var)' ) + .option( + '--spec-version ', + 'Only run conformance scenarios for this spec version' + ) .action(async (options) => { const { owner, repo } = parseRepo(options.repo); let token = options.token || process.env.GITHUB_TOKEN; + let specVersion: SpecVersion | undefined; + if (options.specVersion) { + if (!ALL_SPEC_VERSIONS.includes(options.specVersion as SpecVersion)) { + console.error(`Unknown spec version: ${options.specVersion}`); + console.error(`Valid versions: ${ALL_SPEC_VERSIONS.join(', ')}`); + process.exit(1); + } + specVersion = options.specVersion as SpecVersion; + } + if (!token) { // Try to get token from GitHub CLI try { @@ -90,14 +106,16 @@ export function createTierCheckCommand(): Command { ] = await Promise.all([ checkConformance({ serverUrl: options.conformanceServerUrl, - skip: options.skipConformance + skip: options.skipConformance, + specVersion }).then((r) => { console.error(' ✓ Server Conformance'); return r; }), checkClientConformance({ clientCmd: options.clientCmd, - skip: options.skipConformance || !options.clientCmd + skip: options.skipConformance || !options.clientCmd, + specVersion }).then((r) => { console.error(' ✓ Client Conformance'); return r;