From 9167915281e5624d4436f7ea5d4feedd2dabd69e Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 7 Jul 2026 14:26:00 +0200 Subject: [PATCH] fix(ci): guard eval schema generation --- .agents/verification.md | 5 ++ .github/workflows/ci.yml | 3 + package.json | 2 + packages/core/scripts/generate-eval-schema.ts | 2 +- .../evaluation/validation/eval-file.schema.ts | 5 +- scripts/check-eval-schema.ts | 56 +++++++++++++++++++ 6 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 scripts/check-eval-schema.ts diff --git a/.agents/verification.md b/.agents/verification.md index 0a2b9bb12..22bc5ba89 100644 --- a/.agents/verification.md +++ b/.agents/verification.md @@ -12,6 +12,11 @@ This file expands [AGENTS.md](../AGENTS.md) for testing, manual UAT, CLI and bro smallest command that exercises the changed code, such as a package-specific test, a single test file, `bun run typecheck` for type-facing changes, or `bun run validate:examples` for eval example changes. +- When changing `packages/core/src/evaluation/validation/eval-file.schema.ts`, + run `bun run validate:eval-schema` from the repo root. The command regenerates + `skills-data/agentv-eval-writer/references/eval.schema.json` from the Zod + source and fails with a schema diff if the generated artifact needs to be + committed. Do not edit the generated JSON schema by hand. - Do not default to full local workspace validation before every PR. Push/open the PR so GitHub Actions runs the broad build, typecheck, lint, test, diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5371843fa..3fc33cc5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,6 +166,9 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile + - name: Check generated eval schema sync + run: bun run validate:eval-schema + - name: Run tests run: bun run test diff --git a/package.json b/package.json index 5ffebe9c3..a7b64f7c5 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,8 @@ "agentv:buildrun": "bun run build && bun apps/cli/dist/cli.js", "beads:check": "bun scripts/check-beads-context.ts", "debug:pi-sdk-tools": "bun scripts/debug-pi-sdk-tools.ts", + "generate:eval-schema": "bun packages/core/scripts/generate-eval-schema.ts", + "validate:eval-schema": "bun scripts/check-eval-schema.ts", "validate:promptfoo-export": "bun test scripts/export-promptfoo-config.test.ts", "validate:examples": "EVAL_CRITERIA=placeholder CUSTOM_SYSTEM_PROMPT=placeholder bun scripts/validate-example-evals.ts", "eval:baseline-check": "bun scripts/check-eval-baselines.ts", diff --git a/packages/core/scripts/generate-eval-schema.ts b/packages/core/scripts/generate-eval-schema.ts index 7927be404..5af5beec8 100644 --- a/packages/core/scripts/generate-eval-schema.ts +++ b/packages/core/scripts/generate-eval-schema.ts @@ -2,8 +2,8 @@ import { spawn } from 'node:child_process'; /** * Generates AgentV JSON schemas from Zod schemas. + * Run: bun run generate:eval-schema (from repo root) * Run: bun run generate:schema (from packages/core) - * Or: bun packages/core/scripts/generate-eval-schema.ts (from repo root) */ import { writeFile } from 'node:fs/promises'; import path from 'node:path'; diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index ea6ac79d5..12b72f0f5 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -3,8 +3,9 @@ * Used to generate eval.schema.json for AI agent reference. * * IMPORTANT: This schema describes the YAML input format, not the parsed runtime types. - * When adding new eval features, update this schema AND run `bun run generate:schema` - * to regenerate eval.schema.json. The sync test will fail if they diverge. + * When adding new eval features, update this schema AND run + * `bun run validate:eval-schema` from the repo root. That command regenerates + * eval.schema.json and fails if the generated artifact needs to be committed. */ import { z } from 'zod/v3'; diff --git a/scripts/check-eval-schema.ts b/scripts/check-eval-schema.ts new file mode 100644 index 000000000..7e9ba43cd --- /dev/null +++ b/scripts/check-eval-schema.ts @@ -0,0 +1,56 @@ +#!/usr/bin/env bun +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +const repoRoot = path.resolve(import.meta.dirname, '..'); +const schemaPath = 'skills-data/agentv-eval-writer/references/eval.schema.json'; +const schemaAbsolutePath = path.join(repoRoot, schemaPath); + +async function runOrExit(args: string[], options: { readonly cwd: string }): Promise { + const proc = Bun.spawn(args, { + cwd: options.cwd, + stdout: 'inherit', + stderr: 'inherit', + }); + const exitCode = await proc.exited; + if (exitCode !== 0) { + process.exit(exitCode); + } +} + +async function printSchemaDiff(): Promise { + await runOrExit(['git', 'diff', '--', schemaPath], { cwd: repoRoot }); +} + +async function readSchema(): Promise { + try { + return await readFile(schemaAbsolutePath, 'utf8'); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') { + return null; + } + throw error; + } +} + +const before = await readSchema(); +await runOrExit([process.execPath, 'packages/core/scripts/generate-eval-schema.ts'], { + cwd: repoRoot, +}); +const after = await readSchema(); + +if (before === after) { + console.log(`OK: ${schemaPath} is generated from the Zod source`); + process.exit(0); +} + +const message = + 'Generated eval schema is out of sync. This command regenerated eval.schema.json from packages/core/src/evaluation/validation/eval-file.schema.ts; review and commit the generated diff instead of editing the JSON manually.'; +if (process.env.GITHUB_ACTIONS === 'true') { + console.error(`::error file=${schemaPath}::${message}`); +} +console.error(`[schema] ERROR: ${message}`); +console.error(' Run locally: bun run validate:eval-schema'); +console.error(` Generated file: ${schemaPath}`); +await printSchemaDiff(); +process.exit(1);