diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index b7230ab8..71da650b 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -57,6 +57,8 @@ import { ResourcesNotFoundErrorScenario } from './server/resources'; +import { ResourcesDirectoryReadScenario } from './server/directory'; + import { PromptsListScenario, PromptsGetSimpleScenario, @@ -173,6 +175,9 @@ const allClientScenariosList: ClientScenario[] = [ // Resources error handling (SEP-2164) new ResourcesNotFoundErrorScenario(), + // Skills extension (SEP-2640) — resources/directory/read surface (2e04c48d) + new ResourcesDirectoryReadScenario(), + // Prompts scenarios new PromptsListScenario(), new PromptsGetSimpleScenario(), diff --git a/src/scenarios/server/directory.ts b/src/scenarios/server/directory.ts new file mode 100644 index 00000000..2db1f89d --- /dev/null +++ b/src/scenarios/server/directory.ts @@ -0,0 +1,289 @@ +/** + * SEP-2640 Skills extension scenarios — focused on the resources/directory/read + * surface added in spec commit 2e04c48d (2026-06-09). + * + * One scenario, six checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec quote lives next to its check ID in + * src/seps/sep-2640.yaml, so the YAML and the scenario stay in lock-step. + * + * Capability discovery: the SEP allows multiple shapes for declaring the + * extension; the wire-observable signal we can rely on is whether + * resources/directory/read responds at all. A -32601 method-not-found is the + * only definitive "server didn't declare directoryRead" signal; any other + * response (success, -32602, etc.) means the server registered the method, + * which the SEP requires of any server that declared the capability. + * + * Fixture assumption: the scenario expects the standard mcpkit examples/skills + * fixture which exposes skill://acme/billing/refunds with a templates/ + * subtree containing at least one subdirectory. When the connected server is + * not a skills server (no skill:// resources at all), every check is emitted + * as SKIPPED — keeps the scenario green against the upstream everything-server + * while emitting real verdicts against any skills-capable fixture. + */ + +import { ClientScenario, ConformanceCheck } from '../../types'; +import { JsonRpcError, type RunContext } from '../../connection'; +import type { ListResourcesResult } from '../../spec-types/2025-06-18'; + +interface ResourceLike { + uri: string; + name?: string; + mimeType?: string; +} + +interface DirectoryReadResult { + resources?: ResourceLike[]; + nextCursor?: string; +} + +const SEP_2640_URL = + 'https://modelcontextprotocol.io/seps/2640-skills-extension#directory-listing'; + +const HAPPY_PATH_URI = 'skill://acme/billing/refunds/templates'; +const NON_DIRECTORY_URI = 'skill://acme/billing/refunds/SKILL.md'; + +const SPEC_REFERENCE = [ + { + id: 'SEP-2640-directory-listing', + url: SEP_2640_URL + } +]; + +const JSONRPC_METHOD_NOT_FOUND = -32601; +const JSONRPC_INVALID_PARAMS = -32602; + +function check( + id: string, + description: string, + status: 'SUCCESS' | 'FAILURE' | 'SKIPPED', + extras: Partial = {} +): ConformanceCheck { + return { + id, + name: id, + description, + status, + timestamp: new Date().toISOString(), + specReferences: SPEC_REFERENCE, + ...extras + }; +} + +export class ResourcesDirectoryReadScenario implements ClientScenario { + name = 'sep-2640-skills'; + readonly source = { + extensionId: 'io.modelcontextprotocol/skills' + } as const; + description = `SEP-2640 Skills extension: resources/directory/read surface (added in spec commit 2e04c48d, 2026-06-09). + +**Endpoint**: \`resources/directory/read\` (gated by \`io.modelcontextprotocol/skills.directoryRead: true\`) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-2640.yaml): + +- \`sep-2640-capability-directory-read-flag\` — server effectively declared directoryRead +- \`sep-2640-directory-read-method-registered\` — method registered for served skill directories +- \`sep-2640-directory-read-result-resources-shape\` — result has resources[] of direct children +- \`sep-2640-directory-read-subdir-mimetype\` — subdirectories surface with \`inode/directory\` mime +- \`sep-2640-directory-read-invalid-params\` — non-directory URI returns \`-32602\` +- \`sep-2640-directory-read-pagination\` — \`nextCursor\` round-trips per resources/list contract + +**Fixture expectation**: the server exposes \`skill://acme/billing/refunds/templates\` with at least one subdirectory child. Without any \`skill://\` resources every check emits SKIPPED.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + // SKIP gate: if the server exposes no skill:// resources, treat the + // run as not-applicable rather than failing every check. + let resources: ResourceLike[] = []; + try { + const list = await conn.request('resources/list'); + resources = (list.resources ?? []) as ResourceLike[]; + } catch { + // resources/list missing is itself diagnostic — the server can't be a + // skills server. Fall through to the SKIP branch. + } + const hasSkills = resources.some((r) => r.uri.startsWith('skill://')); + if (!hasSkills) { + const reason = + 'Server exposes no skill:// resources; SEP-2640 directoryRead checks not applicable.'; + return [ + 'sep-2640-capability-directory-read-flag', + 'sep-2640-directory-read-method-registered', + 'sep-2640-directory-read-result-resources-shape', + 'sep-2640-directory-read-subdir-mimetype', + 'sep-2640-directory-read-invalid-params', + 'sep-2640-directory-read-pagination' + ].map((id) => check(id, reason, 'SKIPPED', { errorMessage: reason })); + } + + const checks: ConformanceCheck[] = []; + + // === Happy path: list a known directory === + let happy: DirectoryReadResult | undefined; + let happyErr: unknown; + try { + happy = await conn.request( + 'resources/directory/read', + { uri: HAPPY_PATH_URI } + ); + } catch (e) { + happyErr = e; + } + + const isMethodNotFound = + happyErr instanceof JsonRpcError && + happyErr.code === JSONRPC_METHOD_NOT_FOUND; + + // Check 1: capability declaration (derived from method registration). + checks.push( + check( + 'sep-2640-capability-directory-read-flag', + 'Server declared the directoryRead capability — derived from whether resources/directory/read is registered (a server that did not declare directoryRead would return -32601 method-not-found).', + isMethodNotFound ? 'FAILURE' : 'SUCCESS', + isMethodNotFound + ? { + errorMessage: `resources/directory/read returned -32601, implying the server did not declare directoryRead: ${ + (happyErr as JsonRpcError).message + }` + } + : {} + ) + ); + + // Check 2: method registered. + checks.push( + check( + 'sep-2640-directory-read-method-registered', + 'resources/directory/read accepts a call against a known skill subdirectory.', + happy !== undefined + ? 'SUCCESS' + : isMethodNotFound + ? 'FAILURE' + : 'FAILURE', + happy !== undefined + ? { details: { uri: HAPPY_PATH_URI } } + : { + errorMessage: + happyErr instanceof Error + ? happyErr.message + : String(happyErr) + } + ) + ); + + // Check 3: result shape — resources[] of Resource objects. + const shapeOk = Array.isArray(happy?.resources); + const shapeErrs: string[] = []; + if (!shapeOk) { + shapeErrs.push('result.resources is not an array'); + } else { + happy!.resources!.forEach((r, i) => { + if (typeof r.uri !== 'string') + shapeErrs.push(`resources[${i}].uri is not a string`); + }); + } + checks.push( + check( + 'sep-2640-directory-read-result-resources-shape', + 'Result carries resources[] whose entries match the Resource shape (uri at minimum) from resources/list.', + shapeErrs.length === 0 ? 'SUCCESS' : 'FAILURE', + shapeErrs.length > 0 + ? { errorMessage: shapeErrs.join('; ') } + : { details: { entryCount: happy?.resources?.length ?? 0 } } + ) + ); + + // Check 4: subdirectory mime marker. + const subdirChild = happy?.resources?.find( + (r) => r.mimeType === 'inode/directory' + ); + const hasSubdir = subdirChild !== undefined; + checks.push( + check( + 'sep-2640-directory-read-subdir-mimetype', + 'Subdirectory child carries mimeType "inode/directory" so clients can descend.', + hasSubdir ? 'SUCCESS' : 'FAILURE', + hasSubdir + ? { details: { subdirectoryUri: subdirChild!.uri } } + : { + errorMessage: + 'Expected at least one child with mimeType "inode/directory" under ' + + HAPPY_PATH_URI + + '. Server fixture should expose a subdirectory there.' + } + ) + ); + + // === Error path: non-directory URI === + let invalidParamsOk = false; + let invalidParamsDetail = ''; + try { + await conn.request('resources/directory/read', { + uri: NON_DIRECTORY_URI + }); + invalidParamsDetail = + 'expected -32602 for non-directory URI, got success'; + } catch (e) { + if (e instanceof JsonRpcError && e.code === JSONRPC_INVALID_PARAMS) { + invalidParamsOk = true; + } else if (e instanceof JsonRpcError) { + invalidParamsDetail = `expected -32602 for non-directory URI, got ${e.code}: ${e.message}`; + } else { + invalidParamsDetail = `expected -32602, got non-JsonRpcError: ${ + e instanceof Error ? e.message : String(e) + }`; + } + } + checks.push( + check( + 'sep-2640-directory-read-invalid-params', + 'Non-directory URI yields -32602 Invalid params.', + invalidParamsOk ? 'SUCCESS' : 'FAILURE', + invalidParamsOk ? {} : { errorMessage: invalidParamsDetail } + ) + ); + + // === Pagination contract === + // The SEP is permissive: a single-page response with no nextCursor is + // conformant. The check passes when either (a) the first response has + // no nextCursor at all, or (b) the cursor round-trips on a follow-up + // call. mcpkit's defaultDirectoryReadPageSize = 0 puts it in (a). + let paginationOk = false; + let paginationDetail = ''; + const firstCursor = happy?.nextCursor; + if (!firstCursor) { + paginationOk = true; + paginationDetail = 'single-page response (no nextCursor)'; + } else { + try { + const second = await conn.request( + 'resources/directory/read', + { uri: HAPPY_PATH_URI, cursor: firstCursor } + ); + paginationOk = Array.isArray(second.resources); + paginationDetail = paginationOk + ? `nextCursor round-tripped: ${firstCursor}` + : 'follow-up call returned non-array resources'; + } catch (e) { + paginationDetail = `follow-up call with cursor failed: ${ + e instanceof Error ? e.message : String(e) + }`; + } + } + checks.push( + check( + 'sep-2640-directory-read-pagination', + 'nextCursor round-trips per the resources/list contract (single-page responses are conformant).', + paginationOk ? 'SUCCESS' : 'FAILURE', + paginationOk + ? { details: { paginationDetail } } + : { errorMessage: paginationDetail } + ) + ); + + return checks; + } finally { + await conn.close(); + } + } +} diff --git a/src/seps/sep-2640.yaml b/src/seps/sep-2640.yaml new file mode 100644 index 00000000..024d06c3 --- /dev/null +++ b/src/seps/sep-2640.yaml @@ -0,0 +1,97 @@ +# spec_source: modelcontextprotocol/modelcontextprotocol@556154c088371149c120172e95bb634655f00cbe seps/2640-skills-extension.md +# extracted: 2026-06-05 +# forward_reference: rows sep-2640-capability-directory-read-flag through +# sep-2640-directory-read-pagination track SEP commit +# 2e04c48da90224000e750ffd54a3611f2824fbc0 (2026-06-09) — the +# resources/directory/read addition. The file-level provenance above +# stays at 556154c because the PR 97 schema rewrite (360123d0, +# 2026-06-08) made 3 existing rows stale (sep-2640-index-entry-type-enum, +# sep-2640-index-name-required, sep-2640-index-digest-required) and +# drifted ~11 others' verbatim wording. Full re-extraction at SEP HEAD +# is mcpkit#780's lifecycle; this file deliberately holds at 556154c +# until that lands. +sep: 2640 +spec_url: https://modelcontextprotocol.io/seps/2640-skills-extension#specification +requirements: + - check: sep-2640-skillmd-required + text: 'Every skill MUST contain a `SKILL.md` file at its root.' + - check: sep-2640-skillmd-frontmatter + text: '`SKILL.md` MUST begin with YAML frontmatter containing at minimum the `name` and `description` fields as defined by the Agent Skills specification.' + - check: sep-2640-skill-uri-scheme + text: 'Each file within a skill directory is exposed as an MCP resource. Servers SHOULD use the `skill://` URI scheme, under which the resource URI has the form: `skill:///`' + - check: sep-2640-final-segment-equals-name + text: "The final segment of `` MUST equal the skill's `name` as declared in its `SKILL.md` frontmatter." + - check: sep-2640-no-nested-skills + text: 'A `SKILL.md` MUST NOT appear in any descendant directory of a skill. The skill directory is the boundary; skills do not nest inside other skills.' + - check: sep-2640-name-naming-rules + text: "The final `` segment, being the skill `name`, MUST satisfy the Agent Skills specification's naming rules." + - check: sep-2640-prefix-rfc3986 + text: 'Prefix segments SHOULD be valid URI path segments per RFC 3986; no further constraints are imposed on them.' + - check: sep-2640-skillmd-mimetype + text: 'For each `skill:///SKILL.md` resource: `mimeType` SHOULD be `text/markdown`.' + - check: sep-2640-skillmd-metadata-name + text: 'For each `skill:///SKILL.md` resource: `name` SHOULD be set from the `name` field of the `SKILL.md` YAML frontmatter. By the path constraint above, this will equal the final segment of ``.' + - check: sep-2640-skillmd-metadata-description + text: 'For each `skill:///SKILL.md` resource: `description` SHOULD be set from the `description` field of the `SKILL.md` YAML frontmatter.' + - check: sep-2640-meta-prefix + text: 'When `_meta` keys are used for skill resources, implementations SHOULD use the `io.modelcontextprotocol.skills/` reverse-domain prefix.' + - check: sep-2640-host-load-by-uri + text: 'hosts MUST support loading a skill given only its URI' + - check: sep-2640-server-expose-index + text: 'A server SHOULD expose a resource at the well-known URI `skill://index.json` whose content is a JSON index of the skills it serves.' + - check: sep-2640-index-entry-type-enum + text: '`skills[].type` MUST be `"skill-md"` or `"archive"`.' + - check: sep-2640-index-name-required + text: '`skills[].name` matches the `SKILL.md` frontmatter `name` and the final segment of the skill path.' + - check: sep-2640-index-digest-required + text: '`skills[].digest` is the SHA-256 content digest of the artifact, formatted as `sha256:{hex}` (64 lowercase hex characters).' + - check: sep-2640-client-ignore-unrecognized + text: 'Clients SHOULD ignore unrecognized fields and SHOULD skip entries with an unrecognized `type`.' + - check: sep-2640-archive-format + text: 'the archive MUST be `.tar.gz` (gzip-compressed tar, `mimeType` `application/gzip`) or `.zip` (`mimeType` `application/zip`)' + - check: sep-2640-host-support-archive-formats + text: 'hosts MUST support both `.tar.gz` and `.zip` archive formats' + - check: sep-2640-archive-skillmd-at-root + text: 'Archive contents represent the skill directory directly — `SKILL.md` MUST be at the archive root, not nested inside a wrapper directory' + - check: sep-2640-archive-no-traversal + text: 'the archive MUST NOT contain path-traversal sequences (`..`) or absolute paths' + - check: sep-2640-host-archive-safety + text: 'Hosts unpacking an archive MUST apply the archive safety requirements of the Agent Skills specification: reject archives containing path-traversal sequences or absolute paths, reject symlinks or hard links that resolve outside the skill directory, and enforce a limit on total unpacked size / Hosts MUST validate archives per the Agent Skills archive safety requirements: reject path traversal and absolute paths, reject links resolving outside the skill directory, and bound total unpacked size to prevent decompression bombs.' + - check: sep-2640-host-verify-digest + text: 'Hosts MUST verify retrieved content against the `digest` in the index / hosts MUST NOT use unverified content.' + - check: sep-2640-host-no-empty-index-assumption + text: 'Hosts MUST NOT treat an absent or empty index as proof that a server has no skills.' + + # resources/directory/read additions (SEP commit 2e04c48d, 2026-06-09) + - check: sep-2640-capability-directory-read-flag + text: 'Clients MUST NOT call `resources/directory/read` against a server that has not declared `directoryRead: true`.' + - check: sep-2640-directory-read-method-registered + text: 'A server that declares `directoryRead` MUST support the method for every directory within the skill namespaces it serves as individual files.' + - check: sep-2640-directory-read-subdir-mimetype + text: 'A _directory resource_ is a resource whose `mimeType` is `inode/directory`.' + - check: sep-2640-directory-read-result-resources-shape + text: 'The result contains every direct child of the directory: files with their ordinary resource metadata, subdirectories listed as directory resources (`mimeType: "inode/directory"`). The listing is not recursive; clients descend by calling the method again on a child directory.' + - check: sep-2640-directory-read-invalid-params + text: 'The method applies only to directory resources. If the URI does not exist, or exists but is not a directory resource, the server MUST return error `-32602` (Invalid params) — the same code `resources/read` uses for unknown resources.' + - check: sep-2640-directory-read-pagination + text: 'Pagination mirrors `resources/list`: when the result includes `nextCursor`, the client passes it back as `cursor` to retrieve the next page.' + + - text: 'Per RFC 3986, the first segment of `` occupies the authority component. This carries no special semantics under this convention and clients MUST NOT attempt DNS or network resolution of it.' + excluded: 'DNS and network resolution sit below the MCP wire layer; the harness cannot observe whether the client performed name lookups on URI authority components.' + - text: "[Hosts] SHOULD determine the format from the resource's `mimeType`, falling back to the URL suffix" + excluded: 'Internal decision logic: when `mimeType` and URL suffix agree, the harness cannot distinguish a host that branched on `mimeType` from one that fell back to the suffix.' + - text: 'Hosts MUST treat MCP-served skill content as untrusted model input, subject to the same prompt-injection defenses applied to any server-provided text. A server being connected does not make its skill content authoritative.' + excluded: 'Internal host policy: "treats as untrusted" is an assertion about how content is reasoned over downstream of the read, not about wire traffic.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - text: 'Hosts MUST NOT honor mechanisms in skill content that would cause local code execution without explicit user opt-in. This includes, non-exhaustively: hook declarations, pre/post-invocation scripts, shell commands embedded in frontmatter, or any field that a filesystem-sourced skill might use to register executable behavior on the host.' + excluded: 'Local code execution and explicit user opt-in are host-side filesystem and UX behaviors; not protocol-observable on the wire.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - text: 'Hosts MUST either ignore such fields entirely when the skill arrives over MCP, or gate them behind an explicit per-skill user approval that states what will execute and where.' + excluded: 'Either branch (silent ignore vs. UI-gated approval) is a host-internal handling choice; not protocol-observable.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - text: 'Hosts MUST NOT treat skill resources as higher-authority than other context. Explicit user policy governs whether a skill is loaded at all.' + excluded: 'Context-authority ordering is an internal prompting decision; not protocol-observable.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications + - text: "Hosts SHOULD indicate which server a skill originates from when presenting it, SHOULD let users inspect a skill's content before it is loaded into model context" + excluded: 'UI presentation requirements (origin indicator, pre-load inspection); the harness cannot observe what the host displays to users.' + url: https://modelcontextprotocol.io/seps/2640-skills-extension#security-implications diff --git a/src/types.ts b/src/types.ts index 4da546fe..a4e1980e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -68,7 +68,8 @@ export type ScenarioSpecTag = SpecVersion | 'extension'; */ export const EXTENSION_IDS = [ 'io.modelcontextprotocol/oauth-client-credentials', - 'io.modelcontextprotocol/enterprise-managed-authorization' + 'io.modelcontextprotocol/enterprise-managed-authorization', + 'io.modelcontextprotocol/skills' ] as const; export type ExtensionId = (typeof EXTENSION_IDS)[number];