Save traces option for buffbench

Author: jahooma

evals/buffbench/main-hard-tasks.ts

@@ -13,6 +13,8 @@ function loadTaskIds(evalPath: string): string[] {
 }
 
 async function main() {
+  const saveTraces = process.argv.includes('--save-traces')
+
   const evalPaths = [
     path.join(__dirname, 'eval-codebuff2.json'),
     path.join(__dirname, 'eval-manifold2.json'),
@@ -33,6 +35,7 @@ async function main() {
     agents: ['base2', 'external:claude'],
     taskIds: allTaskIds,
     taskConcurrency: 4,
+    saveTraces,
   })
 
   process.exit(0)

evals/buffbench/main-nightly.ts

@@ -8,6 +8,8 @@ import type { MetaAnalysisResult } from './meta-analyzer'
 import type { AgentEvalResults } from './types'
 
 async function main() {
+  const saveTraces = process.argv.includes('--save-traces')
+
   console.log('Starting nightly buffbench evaluation...')
   console.log('Eval set: codebuff')
   console.log()
@@ -16,6 +18,7 @@ async function main() {
     evalDataPaths: [ path.join(__dirname, 'eval-codebuff.json')],
     agents: ['base2-free'],
     taskConcurrency: 3,
+    saveTraces,
   })
 
   console.log('\nNightly buffbench evaluation completed successfully!')

evals/buffbench/main-single-eval.ts

@@ -3,10 +3,13 @@ import path from 'path'
 import { runBuffBench } from './run-buffbench'
 
 async function main() {
+  const saveTraces = process.argv.includes('--save-traces')
+
   await runBuffBench({
     evalDataPaths: [path.join(__dirname, 'eval-codebuff.json')],
     agents: ['base2'],
     taskIds: ['filter-system-history'],
+    saveTraces,
   })
 
   process.exit(0)

evals/buffbench/main.ts

@@ -3,13 +3,16 @@ import path from 'path'
 import { runBuffBench } from './run-buffbench'
 
 async function main() {
+  const saveTraces = process.argv.includes('--save-traces')
+
   // Compare Codebuff agents against external CLI agents
   // Use 'external:claude' for Claude Code CLI
   // Use 'external:codex' for OpenAI Codex CLI
   await runBuffBench({
     evalDataPaths: [path.join(__dirname, 'eval-codebuff.json')],
     agents: ['base2-free'],
     taskConcurrency: 5,
+    saveTraces,
   })
 
   process.exit(0)

evals/buffbench/run-buffbench.ts

@@ -57,6 +57,7 @@ async function runTask(options: {
   printEvents: boolean
   finalCheckCommands?: string[]
   disableAnalysis?: boolean
+  saveTraces?: boolean
 }) {
   const {
     client,
@@ -74,6 +75,7 @@ async function runTask(options: {
     printEvents,
     finalCheckCommands,
     disableAnalysis,
+    saveTraces = false,
   } = options
 
   console.log(
@@ -173,6 +175,21 @@ async function runTask(options: {
       finalCheckOutputs: agentResult.finalCheckOutputs,
     })
 
+    // Save judge traces to separate files if saveTraces is enabled
+    if (saveTraces) {
+      const tracesDir = path.join(logsDir, 'traces')
+      if (!fs.existsSync(tracesDir)) {
+        fs.mkdirSync(tracesDir, { recursive: true })
+      }
+
+      // Save agent trace only (not judge traces)
+      const agentTracePath = path.join(
+        tracesDir,
+        `${index + 1}-${safeTaskId}-${safeAgentId}-${safeCommitShort}-agent.json`,
+      )
+      fs.writeFileSync(agentTracePath, JSON.stringify(agentResult.trace, null, 2))
+    }
+
     fs.writeFileSync(
       tracePath,
       JSON.stringify(commitTraces[commitTraces.length - 1], null, 2),
@@ -300,6 +317,7 @@ export async function runBuffBench(options: {
   taskIds?: string[]
   extractLessons?: boolean
   disableAnalysis?: boolean
+  saveTraces?: boolean
 }) {
   const {
     evalDataPaths,
@@ -308,6 +326,7 @@ export async function runBuffBench(options: {
     taskIds,
     extractLessons = false,
     disableAnalysis = false,
+    saveTraces = false,
   } = options
 
   if (evalDataPaths.length === 0) {
@@ -453,6 +472,7 @@ export async function runBuffBench(options: {
         printEvents: agents.length === 1 && taskConcurrency === 1,
         finalCheckCommands: evalData.finalCheckCommands,
         disableAnalysis,
+        saveTraces,
       }),
     )
   })