diff --git a/src/index.ts b/src/index.ts index 537fb16..232efa9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,6 +10,10 @@ import { printServerSummary, runInteractiveMode } from './runner'; +import { + printAuthorizationServerSummary, + runAuthorizationServerConformanceTest +} from './runner/authorization-server'; import { listScenarios, listClientScenarios, @@ -23,11 +27,17 @@ import { listScenariosForSpec, listClientScenariosForSpec, getScenarioSpecVersions, + listClientScenariosForAuthorizationServer, + listClientScenariosForAuthorizationServerForSpec, ALL_SPEC_VERSIONS } from './scenarios'; import type { SpecVersion } from './scenarios'; import { ConformanceCheck } from './types'; -import { ClientOptionsSchema, ServerOptionsSchema } from './schemas'; +import { + AuthorizationServerOptionsSchema, + ClientOptionsSchema, + ServerOptionsSchema +} from './schemas'; import { loadExpectedFailures, evaluateBaseline, @@ -52,12 +62,19 @@ function resolveSpecVersion(value: string): SpecVersion { function filterScenariosBySpecVersion( allScenarios: string[], version: SpecVersion, - command: 'client' | 'server' + command: 'client' | 'server' | 'authorization' ): string[] { - const versionScenarios = - command === 'client' - ? listScenariosForSpec(version) - : listClientScenariosForSpec(version); + let versionScenarios: string[]; + if (command === 'client') { + versionScenarios = listScenariosForSpec(version); + } else if (command === 'server') { + versionScenarios = listClientScenariosForSpec(version); + } else if (command === 'authorization') { + versionScenarios = + listClientScenariosForAuthorizationServerForSpec(version); + } else { + versionScenarios = []; + } const allowed = new Set(versionScenarios); return allScenarios.filter((s) => allowed.has(s)); } @@ -444,6 +461,87 @@ program } }); +// Authorization command - tests an authorization server implementation +program + .command('authorization') + .description( + 'Run conformance tests against an authorization server implementation' + ) + .requiredOption('--url ', 'URL of the authorization server to test') + .option('-o, --output-dir ', 'Save results to this directory') + .option( + '--spec-version ', + 'Filter scenarios by spec version (cumulative for date versions)' + ) + .action(async (options) => { + try { + // Validate options with Zod + const validated = AuthorizationServerOptionsSchema.parse(options); + const outputDir = options.outputDir; + const specVersionFilter = options.specVersion + ? resolveSpecVersion(options.specVersion) + : undefined; + + let scenarios: string[]; + scenarios = listClientScenariosForAuthorizationServer(); + if (specVersionFilter) { + scenarios = filterScenariosBySpecVersion( + scenarios, + specVersionFilter, + 'authorization' + ); + } + console.log( + `Running test (${scenarios.length} scenarios) against ${validated.url}\n` + ); + + const allResults: { scenario: string; checks: ConformanceCheck[] }[] = []; + for (const scenarioName of scenarios) { + console.log(`\n=== Running scenario: ${scenarioName} ===`); + try { + const result = await runAuthorizationServerConformanceTest( + validated.url, + scenarioName, + outputDir + ); + allResults.push({ scenario: scenarioName, checks: result.checks }); + } catch (error) { + console.error(`Failed to run scenario ${scenarioName}:`, error); + allResults.push({ + scenario: scenarioName, + checks: [ + { + id: scenarioName, + name: scenarioName, + description: 'Failed to run scenario', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: + error instanceof Error ? error.message : String(error) + } + ] + }); + } + } + const { totalFailed } = printAuthorizationServerSummary(allResults); + process.exit(totalFailed > 0 ? 1 : 0); + } catch (error) { + if (error instanceof ZodError) { + console.error('Validation error:'); + error.errors.forEach((err) => { + console.error(` ${err.path.join('.')}: ${err.message}`); + }); + console.error('\nAvailable authorization server scenarios:'); + listClientScenariosForAuthorizationServer().forEach((s) => + console.error(` - ${s}`) + ); + process.exit(1); + } + console.error('Authorization server test error:', error); + process.exit(1); + } + }); + // Tier check command program.addCommand(createTierCheckCommand()); @@ -453,6 +551,7 @@ program .description('List available test scenarios') .option('--client', 'List client scenarios') .option('--server', 'List server scenarios') + .option('--authorization', 'List authorization server scenarios') .option( '--spec-version ', 'Filter scenarios by spec version (cumulative for date versions)' @@ -462,7 +561,10 @@ program ? resolveSpecVersion(options.specVersion) : undefined; - if (options.server || (!options.client && !options.server)) { + if ( + options.server || + (!options.client && !options.server && !options.authorization) + ) { console.log('Server scenarios (test against a server):'); let serverScenarios = listClientScenarios(); if (specVersionFilter) { @@ -478,7 +580,10 @@ program }); } - if (options.client || (!options.client && !options.server)) { + if ( + options.client || + (!options.client && !options.server && !options.authorization) + ) { if (options.server || (!options.client && !options.server)) { console.log(''); } @@ -496,6 +601,31 @@ program console.log(` - ${s}${v ? ` [${v}]` : ''}`); }); } + + if ( + options.authorization || + (!options.authorization && !options.server && !options.client) + ) { + if (!(options.authorization && !options.server && !options.client)) { + console.log(''); + } + console.log( + 'Authorization server scenarios (test against an authorization server):' + ); + let authorizationServerScenarios = + listClientScenariosForAuthorizationServer(); + if (specVersionFilter) { + authorizationServerScenarios = filterScenariosBySpecVersion( + authorizationServerScenarios, + specVersionFilter, + 'authorization' + ); + } + authorizationServerScenarios.forEach((s) => { + const v = getScenarioSpecVersions(s); + console.log(` - ${s}${v ? ` [${v}]` : ''}`); + }); + } }); program.parse(); diff --git a/src/runner/authorization-server.ts b/src/runner/authorization-server.ts new file mode 100644 index 0000000..5b4094b --- /dev/null +++ b/src/runner/authorization-server.ts @@ -0,0 +1,74 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import { ConformanceCheck } from '../types'; +import { getClientScenarioForAuthorizationServer } from '../scenarios'; +import { createResultDir } from './utils'; + +export async function runAuthorizationServerConformanceTest( + serverUrl: string, + scenarioName: string, + outputDir?: string +): Promise<{ + checks: ConformanceCheck[]; + resultDir?: string; + scenarioDescription: string; +}> { + let resultDir: string | undefined; + + if (outputDir) { + resultDir = createResultDir( + outputDir, + scenarioName, + 'authorization-server' + ); + await fs.mkdir(resultDir, { recursive: true }); + } + + // Scenario is guaranteed to exist by CLI validation + const scenario = getClientScenarioForAuthorizationServer(scenarioName)!; + + console.log( + `Running client scenario for authorization server '${scenarioName}' against server: ${serverUrl}` + ); + + const checks = await scenario.run(serverUrl); + + if (resultDir) { + await fs.writeFile( + path.join(resultDir, 'checks.json'), + JSON.stringify(checks, null, 2) + ); + + console.log(`Results saved to ${resultDir}`); + } + + return { + checks, + resultDir, + scenarioDescription: scenario.description + }; +} + +export function printAuthorizationServerSummary( + allResults: { scenario: string; checks: ConformanceCheck[] }[] +): { totalPassed: number; totalFailed: number } { + console.log('\n\n=== SUMMARY ==='); + let totalPassed = 0; + let totalFailed = 0; + + for (const result of allResults) { + const passed = result.checks.filter((c) => c.status === 'SUCCESS').length; + const failed = result.checks.filter((c) => c.status === 'FAILURE').length; + totalPassed += passed; + totalFailed += failed; + + const status = failed === 0 ? '✓' : '✗'; + console.log( + `${status} ${result.scenario}: ${passed} passed, ${failed} failed` + ); + } + + console.log(`\nTotal: ${totalPassed} passed, ${totalFailed} failed`); + + return { totalPassed, totalFailed }; +} diff --git a/src/scenarios/authorization-server/authorization-server-metadata.test.ts b/src/scenarios/authorization-server/authorization-server-metadata.test.ts new file mode 100644 index 0000000..e37b6f9 --- /dev/null +++ b/src/scenarios/authorization-server/authorization-server-metadata.test.ts @@ -0,0 +1,52 @@ +import { describe, it, expect, vi } from 'vitest'; +import { AuthorizationServerMetadataEndpointScenario } from './authorization-server-metadata.js'; +import { request } from 'undici'; + +vi.mock('undici', () => ({ + request: vi.fn() +})); + +const mockedRequest = vi.mocked(request); + +describe('AuthorizationServerMetadataEndpointScenario (SUCCESS only)', () => { + it('returns SUCCESS for valid authorization server metadata', async () => { + const scenario = new AuthorizationServerMetadataEndpointScenario(); + const serverUrl = + 'https://example.com/.well-known/oauth-authorization-server'; + + mockedRequest.mockResolvedValue({ + statusCode: 200, + headers: { + 'content-type': 'application/json' + }, + body: { + json: async () => ({ + issuer: 'https://example.com', + authorization_endpoint: 'https://example.com/auth', + token_endpoint: 'https://example.com/token', + response_types_supported: ['code'] + }) + } + } as any); + + const checks = await scenario.run(serverUrl); + + expect(checks).toHaveLength(1); + + const check = checks[0]; + expect(check.status).toBe('SUCCESS'); + expect(check.errorMessage).toBeUndefined(); + expect(check.details).toBeDefined(); + expect(check.details!.contentType).toContain('application/json'); + expect((check.details!.body as any).issuer).toBe('https://example.com'); + expect((check.details!.body as any).authorization_endpoint).toBe( + 'https://example.com/auth' + ); + expect((check.details!.body as any).token_endpoint).toBe( + 'https://example.com/token' + ); + expect((check.details!.body as any).response_types_supported).toEqual([ + 'code' + ]); + }); +}); diff --git a/src/scenarios/authorization-server/authorization-server-metadata.ts b/src/scenarios/authorization-server/authorization-server-metadata.ts new file mode 100644 index 0000000..2f9a42d --- /dev/null +++ b/src/scenarios/authorization-server/authorization-server-metadata.ts @@ -0,0 +1,138 @@ +/** + * Authorization server metadata endpoint test scenarios for MCP authorization servers + */ +import { + ClientScenarioForAuthorizationServer, + ConformanceCheck, + SpecVersion +} from '../../types'; +import { request } from 'undici'; + +type Status = 'SUCCESS' | 'FAILURE'; + +export class AuthorizationServerMetadataEndpointScenario + implements ClientScenarioForAuthorizationServer +{ + name = 'authorization-server-metadata-endpoint'; + specVersions: SpecVersion[] = ['2025-03-26', '2025-06-18', '2025-11-25']; + description = `Test authorization server metadata endpoint. + +**Authorization Server Implementation Requirements:** + +**Endpoint**: \`authorization server metadata\` + +**Requirements**: +- HTTP response status code MUST be 200 OK +- Content-Type header MUST be application/json +- Return a JSON response including issuer, authorization_endpoint, token_endpoint and response_types_supported +- The issuer value MUST match the URI obtained by removing the well-known URI string from the authorization server metadata URI.`; + + async run(serverUrl: string): Promise { + let status: Status = 'SUCCESS'; + let errorMessage: string | undefined; + let details: any; + + try { + this.validateWellKnownPath(serverUrl); + + const response = await request(serverUrl, { method: 'GET' }); + this.validateStatusCode(response.statusCode); + this.validateContentType(response.headers['content-type']); + + const body = await this.parseJson(response); + this.validateMetadataBody(body, serverUrl); + + details = { + contentType: response.headers['content-type'], + body + }; + } catch (error) { + status = 'FAILURE'; + errorMessage = error instanceof Error ? error.message : String(error); + } + + return [ + { + id: 'authorization-server-metadata', + name: 'AuthorizationServerMetadata', + description: 'Valid authorization server metadata response', + status, + timestamp: new Date().toISOString(), + errorMessage, + specReferences: [ + { + id: 'Authorization-Server-Metadata', + url: 'https://datatracker.ietf.org/doc/html/rfc8414' + } + ], + ...(details ? { details } : {}) + } + ]; + } + + private validateWellKnownPath(serverUrl: string): void { + const url = new URL(serverUrl); + const valid = + url.pathname === '/.well-known/oauth-authorization-server' || + url.pathname.startsWith('/.well-known/oauth-authorization-server/'); + + if (!valid) { + throw new Error(`Invalid path: ${url.pathname}`); + } + } + + private validateStatusCode(statusCode: number): void { + if (statusCode !== 200) { + throw new Error(`Invalid status code: ${statusCode}`); + } + } + + private validateContentType(contentType?: string | string[]): void { + const valid = + typeof contentType === 'string' && + contentType.toLowerCase().includes('application/json'); + + if (!valid) { + throw new Error(`Invalid Content-Type: ${contentType ?? '(missing)'}`); + } + } + + private async parseJson(response: any): Promise> { + const body = await response.body.json(); + if (typeof body !== 'object' || body === null) { + throw new Error('Response body is not an object'); + } + return body; + } + + private validateMetadataBody( + body: Record, + serverUrl: string + ): void { + this.assertString(body.authorization_endpoint, 'authorization_endpoint'); + this.assertString(body.token_endpoint, 'token_endpoint'); + + if ( + !Array.isArray(body.response_types_supported) || + body.response_types_supported.length === 0 + ) { + throw new Error( + 'Response body does not include valid "response_types_supported" claim' + ); + } + + const expectedIssuer = serverUrl.replace( + '/.well-known/oauth-authorization-server', + '' + ); + if (body.issuer !== expectedIssuer) { + throw new Error(`Invalid issuer: ${body.issuer ?? '(missing)'}`); + } + } + + private assertString(value: unknown, name: string): void { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`Response body does not include valid "${name}" claim`); + } + } +} diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index d67fae4..ce3f19e 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -1,4 +1,9 @@ -import { Scenario, ClientScenario, SpecVersion } from '../types'; +import { + Scenario, + ClientScenario, + ClientScenarioForAuthorizationServer, + SpecVersion +} from '../types'; import { InitializeScenario } from './client/initialize'; import { ToolsCallScenario } from './client/tools_call'; import { ElicitationClientDefaultsScenario } from './client/elicitation-defaults'; @@ -59,6 +64,7 @@ import { extensionScenariosList } from './client/auth/index'; import { listMetadataScenarios } from './client/auth/discovery-metadata'; +import { AuthorizationServerMetadataEndpointScenario } from './authorization-server/authorization-server-metadata'; // Pending client scenarios (not yet fully tested/implemented) const pendingClientScenariosList: ClientScenario[] = [ @@ -141,6 +147,23 @@ export const clientScenarios = new Map( allClientScenariosList.map((scenario) => [scenario.name, scenario]) ); +// All client scenarios for authorization server +const allClientScenariosListForAuthorizationServer: ClientScenario[] = [ + // Authorization server scenarios + new AuthorizationServerMetadataEndpointScenario() +]; + +// Client scenarios map for authorization server - built from list +export const clientScenariosForAuthorizationServer = new Map< + string, + ClientScenario +>( + allClientScenariosListForAuthorizationServer.map((scenario) => [ + scenario.name, + scenario + ]) +); + // All client test scenarios (core + backcompat + extensions) const scenariosList: Scenario[] = [ new InitializeScenario(), @@ -178,6 +201,12 @@ export function getClientScenario(name: string): ClientScenario | undefined { return clientScenarios.get(name); } +export function getClientScenarioForAuthorizationServer( + name: string +): ClientScenarioForAuthorizationServer | undefined { + return clientScenariosForAuthorizationServer.get(name); +} + export function listScenarios(): string[] { return Array.from(scenarios.keys()); } @@ -210,6 +239,10 @@ export function listBackcompatScenarios(): string[] { return backcompatScenariosList.map((scenario) => scenario.name); } +export function listClientScenariosForAuthorizationServer(): string[] { + return Array.from(clientScenariosForAuthorizationServer.keys()); +} + export { listMetadataScenarios }; // All valid spec versions, used by the CLI to validate --spec-version input. @@ -233,11 +266,21 @@ export function listClientScenariosForSpec(version: SpecVersion): string[] { .map((s) => s.name); } +export function listClientScenariosForAuthorizationServerForSpec( + version: SpecVersion +): string[] { + return allClientScenariosListForAuthorizationServer + .filter((s) => s.specVersions.includes(version)) + .map((s) => s.name); +} + export function getScenarioSpecVersions( name: string ): SpecVersion[] | undefined { return ( - scenarios.get(name)?.specVersions ?? clientScenarios.get(name)?.specVersions + scenarios.get(name)?.specVersions ?? + clientScenarios.get(name)?.specVersions ?? + clientScenariosForAuthorizationServer.get(name)?.specVersions ); } diff --git a/src/schemas.ts b/src/schemas.ts index 5e81c8c..3143a91 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -44,6 +44,15 @@ export const ServerOptionsSchema = z.object({ export type ServerOptions = z.infer; +// Authorization server command options schema +export const AuthorizationServerOptionsSchema = z.object({ + url: z.string().url('Invalid authorization server URL') +}); + +export type AuthorizationServerOptions = z.infer< + typeof AuthorizationServerOptionsSchema +>; + // Interactive command options schema export const InteractiveOptionsSchema = z.object({ scenario: z diff --git a/src/types.ts b/src/types.ts index c086e65..193686f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -60,3 +60,10 @@ export interface ClientScenario { specVersions: SpecVersion[]; run(serverUrl: string): Promise; } + +export interface ClientScenarioForAuthorizationServer { + name: string; + description: string; + specVersions: SpecVersion[]; + run(serverUrl: string): Promise; +}