From 9179d61cb54d9be0f75ed7f18b89b19834404c34 Mon Sep 17 00:00:00 2001 From: Harrison Weinstock Date: Thu, 30 Apr 2026 21:27:05 +0000 Subject: [PATCH] feat: wire telemetry into all remove.* commands --- integ-tests/add-remove-ab-test.test.ts | 19 +++- integ-tests/add-remove-config-bundle.test.ts | 46 +++++---- integ-tests/add-remove-evaluator.test.ts | 12 ++- integ-tests/add-remove-gateway.test.ts | 32 ++++++- integ-tests/add-remove-policy.test.ts | 17 +++- integ-tests/add-remove-resources.test.ts | 54 +++++++++-- src/cli/commands/remove/command.tsx | 10 +- src/cli/primitives/ABTestPrimitive.ts | 5 +- src/cli/primitives/AgentPrimitive.tsx | 4 +- src/cli/primitives/BasePrimitive.ts | 8 +- src/cli/primitives/CredentialPrimitive.tsx | 4 +- src/cli/primitives/EvaluatorPrimitive.ts | 4 +- src/cli/primitives/GatewayPrimitive.ts | 6 +- src/cli/primitives/GatewayTargetPrimitive.ts | 8 +- src/cli/primitives/MemoryPrimitive.tsx | 4 +- .../primitives/OnlineEvalConfigPrimitive.ts | 4 +- src/cli/primitives/PolicyEnginePrimitive.ts | 8 +- src/cli/primitives/PolicyPrimitive.ts | 6 +- .../primitives/RuntimeEndpointPrimitive.ts | 8 +- src/cli/telemetry/cli-command-run.ts | 96 ++++++++++--------- src/cli/telemetry/schemas/command-run.ts | 20 ++++ src/cli/tui/hooks/useCreateEvaluator.ts | 4 +- src/cli/tui/hooks/useCreateMcp.ts | 4 +- src/cli/tui/hooks/useCreateMemory.ts | 4 +- src/cli/tui/hooks/useCreateOnlineEval.ts | 4 +- src/cli/tui/hooks/useRemove.ts | 8 +- src/cli/tui/screens/agent/useAddAgent.ts | 4 +- .../tui/screens/identity/useCreateIdentity.ts | 4 +- src/cli/tui/screens/policy/AddPolicyFlow.tsx | 6 +- src/cli/tui/screens/remove/useRemoveFlow.ts | 29 ++++-- .../AddRuntimeEndpointFlow.tsx | 4 +- 31 files changed, 310 insertions(+), 136 deletions(-) diff --git a/integ-tests/add-remove-ab-test.test.ts b/integ-tests/add-remove-ab-test.test.ts index 551c86010..1fd1aa7bc 100644 --- a/integ-tests/add-remove-ab-test.test.ts +++ b/integ-tests/add-remove-ab-test.test.ts @@ -5,8 +5,11 @@ import { readProjectConfig, runCLI, } from '../src/test-utils/index.js'; +import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +const telemetry = createTelemetryHelper(); + async function runSuccess(args: string[], cwd: string) { const result = await runCLI(args, cwd); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -38,6 +41,7 @@ describe('integration: add and remove ab-test', () => { afterAll(async () => { await project.cleanup(); + telemetry.destroy(); }); it('requires --name for JSON mode', async () => { @@ -154,17 +158,26 @@ describe('integration: add and remove ab-test', () => { }); it('removes ab-test', async () => { - const json = await runSuccess(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath); + const result = await runCLI(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath, { + env: telemetry.env, + }); + expect(result.exitCode).toBe(0); + const json = JSON.parse(result.stdout); expect(json.success).toBe(true); - // Verify removal from agentcore.json const spec = await readProjectConfig(project.projectPath); const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'MyIntegTest'); expect(abTest).toBeUndefined(); + telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'success' }); }); it('remove returns error for non-existent test', async () => { - const json = await runFailure(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath); + const result = await runCLI(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath, { + env: telemetry.env, + }); + expect(result.exitCode).toBe(1); + const json = JSON.parse(result.stdout); expect(json.error).toContain('not found'); + telemetry.assertMetricEmitted({ command: 'remove.ab-test', exit_reason: 'failure' }); }); }); diff --git a/integ-tests/add-remove-config-bundle.test.ts b/integ-tests/add-remove-config-bundle.test.ts index bd53e7f31..c6c37c257 100644 --- a/integ-tests/add-remove-config-bundle.test.ts +++ b/integ-tests/add-remove-config-bundle.test.ts @@ -7,10 +7,13 @@ import { runFailure, runSuccess, } from '../src/test-utils/index.js'; +import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +const telemetry = createTelemetryHelper(); + describe('integration: add and remove config-bundle', () => { let project: TestProject; @@ -20,6 +23,7 @@ describe('integration: add and remove config-bundle', () => { afterAll(async () => { await project.cleanup(); + telemetry.destroy(); }); // ── Add lifecycle ───────────────────────────────────────────────────── @@ -40,7 +44,7 @@ describe('integration: add and remove config-bundle', () => { expect(json.bundleName).toBe('InlineBundle'); const config = await readProjectConfig(project.projectPath); - const bundle = config.configBundles!.find(b => b.name === 'InlineBundle'); + const bundle = config.configBundles.find(b => b.name === 'InlineBundle'); expect(bundle).toBeDefined(); expect(bundle!.type).toBe('ConfigurationBundle'); expect(bundle!.branchName).toBe('mainline'); @@ -68,7 +72,7 @@ describe('integration: add and remove config-bundle', () => { expect(json.bundleName).toBe('FileBundle'); const config = await readProjectConfig(project.projectPath); - const bundle = config.configBundles!.find(b => b.name === 'FileBundle'); + const bundle = config.configBundles.find(b => b.name === 'FileBundle'); expect(bundle).toBeDefined(); expect(Object.keys(bundle!.components)).toHaveLength(2); }); @@ -102,7 +106,7 @@ describe('integration: add and remove config-bundle', () => { expect(json.bundleName).toBe('FullOptsBundle'); const config = await readProjectConfig(project.projectPath); - const bundle = config.configBundles!.find(b => b.name === 'FullOptsBundle'); + const bundle = config.configBundles.find(b => b.name === 'FullOptsBundle'); expect(bundle).toBeDefined(); expect(bundle!.description).toBe('A bundle with all optional fields'); expect(bundle!.branchName).toBe('feature-branch'); @@ -127,7 +131,7 @@ describe('integration: add and remove config-bundle', () => { expect(json.bundleName).toBe('PlaceholderBundle'); const config = await readProjectConfig(project.projectPath); - const bundle = config.configBundles!.find(b => b.name === 'PlaceholderBundle'); + const bundle = config.configBundles.find(b => b.name === 'PlaceholderBundle'); expect(bundle).toBeDefined(); const keys = Object.keys(bundle!.components); expect(keys).toContain('{{runtime:AgentA}}'); @@ -228,37 +232,45 @@ describe('integration: add and remove config-bundle', () => { describe('remove config-bundle', () => { it('removes an existing config bundle', async () => { - const json = await runSuccess( + const result = await runCLI( ['remove', 'config-bundle', '--name', 'InlineBundle', '--json'], - project.projectPath + project.projectPath, + { env: telemetry.env } ); + expect(result.exitCode).toBe(0); + const json = JSON.parse(result.stdout); expect(json.success).toBe(true); const config = await readProjectConfig(project.projectPath); - const bundle = config.configBundles!.find(b => b.name === 'InlineBundle'); + const bundle = config.configBundles.find(b => b.name === 'InlineBundle'); expect(bundle).toBeUndefined(); + telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'success' }); }); it('returns error for non-existent bundle', async () => { - const json = await runFailure( + const result = await runCLI( ['remove', 'config-bundle', '--name', 'DoesNotExist', '--json'], - project.projectPath + project.projectPath, + { env: telemetry.env } ); + expect(result.exitCode).toBe(1); + const json = JSON.parse(result.stdout); expect(json.error).toContain('not found'); + telemetry.assertMetricEmitted({ command: 'remove.config-bundle', exit_reason: 'failure' }); }); it('removes all remaining config bundles one by one', async () => { const configBefore = await readProjectConfig(project.projectPath); - const remaining = configBefore.configBundles!.map(b => b.name); + const remaining = configBefore.configBundles.map(b => b.name); for (const name of remaining) { await runSuccess(['remove', 'config-bundle', '--name', name, '--json'], project.projectPath); } const configAfter = await readProjectConfig(project.projectPath); - expect(configAfter.configBundles!).toHaveLength(0); + expect(configAfter.configBundles).toHaveLength(0); }); }); @@ -282,10 +294,10 @@ describe('integration: add and remove config-bundle', () => { } const config = await readProjectConfig(project.projectPath); - expect(config.configBundles!).toHaveLength(bundleNames.length); + expect(config.configBundles).toHaveLength(bundleNames.length); for (const name of bundleNames) { - expect(config.configBundles!.find(b => b.name === name)).toBeDefined(); + expect(config.configBundles.find(b => b.name === name)).toBeDefined(); } }); @@ -293,10 +305,10 @@ describe('integration: add and remove config-bundle', () => { await runSuccess(['remove', 'config-bundle', '--name', 'BundleBeta', '--json'], project.projectPath); const config = await readProjectConfig(project.projectPath); - expect(config.configBundles!).toHaveLength(2); - expect(config.configBundles!.find(b => b.name === 'BundleAlpha')).toBeDefined(); - expect(config.configBundles!.find(b => b.name === 'BundleGamma')).toBeDefined(); - expect(config.configBundles!.find(b => b.name === 'BundleBeta')).toBeUndefined(); + expect(config.configBundles).toHaveLength(2); + expect(config.configBundles.find(b => b.name === 'BundleAlpha')).toBeDefined(); + expect(config.configBundles.find(b => b.name === 'BundleGamma')).toBeDefined(); + expect(config.configBundles.find(b => b.name === 'BundleBeta')).toBeUndefined(); }); afterAll(async () => { diff --git a/integ-tests/add-remove-evaluator.test.ts b/integ-tests/add-remove-evaluator.test.ts index 30c3917c3..cee29f858 100644 --- a/integ-tests/add-remove-evaluator.test.ts +++ b/integ-tests/add-remove-evaluator.test.ts @@ -1,10 +1,13 @@ import { createTestProject, parseJsonOutput, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import type { TestProject } from '../src/test-utils/index.js'; +import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +const telemetry = createTelemetryHelper(); + /** Run a CLI command and assert it succeeds, returning parsed JSON output. */ async function runSuccess(args: string[], cwd: string) { - const result = await runCLI(args, cwd); + const result = await runCLI(args, cwd, { env: telemetry.env }); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); const json: unknown = parseJsonOutput(result.stdout); expect(json).toHaveProperty('success', true); @@ -13,7 +16,7 @@ async function runSuccess(args: string[], cwd: string) { /** Run a CLI command and assert it fails, returning parsed JSON output. */ async function runFailure(args: string[], cwd: string) { - const result = await runCLI(args, cwd); + const result = await runCLI(args, cwd, { env: telemetry.env }); expect(result.exitCode).toBe(1); const json: unknown = parseJsonOutput(result.stdout); expect(json).toHaveProperty('success', false); @@ -35,6 +38,7 @@ describe('integration: add and remove evaluators and online eval configs', () => afterAll(async () => { await project.cleanup(); + telemetry.destroy(); }); describe('evaluator and online eval lifecycle', () => { @@ -123,6 +127,7 @@ describe('integration: add and remove evaluators and online eval configs', () => const config = await readProjectConfig(project.projectPath); expect(config.onlineEvalConfigs.find(c => c.name === configName)).toBeUndefined(); + telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'success' }); }); it('removes the evaluator after online eval is gone', async () => { @@ -130,6 +135,7 @@ describe('integration: add and remove evaluators and online eval configs', () => const config = await readProjectConfig(project.projectPath); expect(config.evaluators.find(e => e.name === evalName)).toBeUndefined(); + telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'success' }); }); }); @@ -137,11 +143,13 @@ describe('integration: add and remove evaluators and online eval configs', () => it('fails to remove non-existent evaluator', async () => { const json = await runFailure(['remove', 'evaluator', '--name', 'NonExistent', '--json'], project.projectPath); expect(json.error).toContain('not found'); + telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'failure' }); }); it('fails to remove non-existent online eval config', async () => { const json = await runFailure(['remove', 'online-eval', '--name', 'NonExistent', '--json'], project.projectPath); expect(json.error).toContain('not found'); + telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'failure' }); }); it('rejects evaluator with missing --level', async () => { diff --git a/integ-tests/add-remove-gateway.test.ts b/integ-tests/add-remove-gateway.test.ts index ba4753c9b..8453c5e60 100644 --- a/integ-tests/add-remove-gateway.test.ts +++ b/integ-tests/add-remove-gateway.test.ts @@ -1,9 +1,12 @@ import { createTestProject, runCLI } from '../src/test-utils/index.js'; import type { TestProject } from '../src/test-utils/index.js'; +import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { mkdir, readFile, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +const telemetry = createTelemetryHelper(); + async function readProjectConfig(projectPath: string) { return JSON.parse(await readFile(join(projectPath, 'agentcore/agentcore.json'), 'utf-8')); } @@ -19,6 +22,7 @@ describe('integration: add and remove gateway with external MCP server', () => { afterAll(async () => { await project.cleanup(); + telemetry.destroy(); }); describe('gateway lifecycle', () => { @@ -64,7 +68,9 @@ describe('integration: add and remove gateway with external MCP server', () => { }); it('removes the gateway target', async () => { - const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath); + const result = await runCLI(['remove', 'gateway-target', '--name', targetName, '--json'], project.projectPath, { + env: telemetry.env, + }); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); const json = JSON.parse(result.stdout); @@ -75,10 +81,13 @@ describe('integration: add and remove gateway with external MCP server', () => { const targets = gateway?.targets ?? []; const found = targets.find((t: { name: string }) => t.name === targetName); expect(found, `Target "${targetName}" should be removed`).toBeFalsy(); + telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'success' }); }); it('removes the gateway', async () => { - const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath); + const result = await runCLI(['remove', 'gateway', '--name', gatewayName, '--json'], project.projectPath, { + env: telemetry.env, + }); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); const json = JSON.parse(result.stdout); @@ -88,6 +97,25 @@ describe('integration: add and remove gateway with external MCP server', () => { const gateways = mcpSpec.agentCoreGateways ?? []; const found = gateways.find((g: { name: string }) => g.name === gatewayName); expect(found, `Gateway "${gatewayName}" should be removed`).toBeFalsy(); + telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'success' }); + }); + + it('fails to remove non-existent gateway', async () => { + const result = await runCLI(['remove', 'gateway', '--name', 'NonExistent', '--json'], project.projectPath, { + env: telemetry.env, + }); + expect(result.exitCode).toBe(1); + telemetry.assertMetricEmitted({ command: 'remove.gateway', exit_reason: 'failure' }); + }); + + it('fails to remove non-existent gateway target', async () => { + const result = await runCLI( + ['remove', 'gateway-target', '--name', 'NonExistent', '--json'], + project.projectPath, + { env: telemetry.env } + ); + expect(result.exitCode).toBe(1); + telemetry.assertMetricEmitted({ command: 'remove.gateway-target', exit_reason: 'failure' }); }); }); }); diff --git a/integ-tests/add-remove-policy.test.ts b/integ-tests/add-remove-policy.test.ts index 2830b792a..a44f51c5e 100644 --- a/integ-tests/add-remove-policy.test.ts +++ b/integ-tests/add-remove-policy.test.ts @@ -1,7 +1,10 @@ import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import type { TestProject } from '../src/test-utils/index.js'; +import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +const telemetry = createTelemetryHelper(); + describe('integration: add and remove policy engines and policies', () => { let project: TestProject; @@ -16,6 +19,7 @@ describe('integration: add and remove policy engines and policies', () => { afterAll(async () => { await project.cleanup(); + telemetry.destroy(); }); describe('policy engine lifecycle', () => { @@ -111,7 +115,8 @@ describe('integration: add and remove policy engines and policies', () => { it('removes a policy with --engine flag', async () => { const result = await runCLI( ['remove', 'policy', '--name', policyName, '--engine', engineName, '--json'], - project.projectPath + project.projectPath, + { env: telemetry.env } ); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -125,6 +130,7 @@ describe('integration: add and remove policy engines and policies', () => { expect(engine, `Engine "${engineName}" should still exist`).toBeDefined(); const policy = engine!.policies.find(p => p.name === policyName); expect(policy, `Policy "${policyName}" should be removed`).toBeUndefined(); + telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'success' }); }); }); @@ -272,24 +278,29 @@ describe('integration: add and remove policy engines and policies', () => { }); it('fails to remove non-existent policy', async () => { - const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath); + const result = await runCLI(['remove', 'policy', '--name', 'NonExistentPolicy', '--json'], project.projectPath, { + env: telemetry.env, + }); expect(result.exitCode).toBe(1); const json = JSON.parse(result.stdout); expect(json.success).toBe(false); expect(json.error).toContain('not found'); + telemetry.assertMetricEmitted({ command: 'remove.policy', exit_reason: 'failure' }); }); it('fails to remove non-existent policy engine', async () => { const result = await runCLI( ['remove', 'policy-engine', '--name', 'NonExistentEngine', '--json'], - project.projectPath + project.projectPath, + { env: telemetry.env } ); expect(result.exitCode).toBe(1); const json = JSON.parse(result.stdout); expect(json.success).toBe(false); expect(json.error).toContain('not found'); + telemetry.assertMetricEmitted({ command: 'remove.policy-engine', exit_reason: 'failure' }); }); it('requires --engine when adding a policy', async () => { diff --git a/integ-tests/add-remove-resources.test.ts b/integ-tests/add-remove-resources.test.ts index a89c761dd..19d607481 100644 --- a/integ-tests/add-remove-resources.test.ts +++ b/integ-tests/add-remove-resources.test.ts @@ -41,7 +41,6 @@ describe('integration: add and remove resources', () => { const found = memories!.some((m: Record) => m.name === memoryName); expect(found, `Memory "${memoryName}" should be in config`).toBe(true); - // Verify telemetry telemetry.assertMetricEmitted({ command: 'add.memory', exit_reason: 'success' }); }); @@ -71,7 +70,6 @@ describe('integration: add and remove resources', () => { expect(episodic!.reflectionNamespaces, 'Should have reflectionNamespaces').toBeDefined(); expect(episodic!.reflectionNamespaces!.length).toBeGreaterThan(0); - // Verify telemetry telemetry.assertMetricEmitted({ command: 'add.memory', exit_reason: 'success', @@ -84,7 +82,9 @@ describe('integration: add and remove resources', () => { }); it('removes the memory resource', async () => { - const result = await runCLI(['remove', 'memory', '--name', memoryName, '--json'], project.projectPath); + const result = await runCLI(['remove', 'memory', '--name', memoryName, '--json'], project.projectPath, { + env: telemetry.env, + }); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); const json = JSON.parse(result.stdout); @@ -95,6 +95,8 @@ describe('integration: add and remove resources', () => { const memories = (config.memories as Record[] | undefined) ?? []; const found = memories.some((m: Record) => m.name === memoryName); expect(found, `Memory "${memoryName}" should be removed from config`).toBe(false); + + telemetry.assertMetricEmitted({ command: 'remove.memory', exit_reason: 'success' }); }); }); @@ -119,7 +121,6 @@ describe('integration: add and remove resources', () => { const found = credentials!.some((c: Record) => c.name === credentialName); expect(found, `Credential "${credentialName}" should be in config`).toBe(true); - // Verify telemetry telemetry.assertMetricEmitted({ command: 'add.credential', exit_reason: 'success', @@ -128,7 +129,9 @@ describe('integration: add and remove resources', () => { }); it('removes the credential resource', async () => { - const result = await runCLI(['remove', 'credential', '--name', credentialName, '--json'], project.projectPath); + const result = await runCLI(['remove', 'credential', '--name', credentialName, '--json'], project.projectPath, { + env: telemetry.env, + }); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); const json = JSON.parse(result.stdout); @@ -139,6 +142,8 @@ describe('integration: add and remove resources', () => { const credentials = (config.credentials as Record[] | undefined) ?? []; const found = credentials.some((c: Record) => c.name === credentialName); expect(found, `Credential "${credentialName}" should be removed from config`).toBe(false); + + telemetry.assertMetricEmitted({ command: 'remove.credential', exit_reason: 'success' }); }); }); @@ -162,9 +167,46 @@ describe('integration: add and remove resources', () => { }); it('removes the policy engine resource', async () => { - const result = await runCLI(['remove', 'policy-engine', '--name', engineName, '--json'], project.projectPath); + const result = await runCLI(['remove', 'policy-engine', '--name', engineName, '--json'], project.projectPath, { + env: telemetry.env, + }); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + telemetry.assertMetricEmitted({ command: 'remove.policy-engine', exit_reason: 'success' }); + }); + }); + + describe('remove failure telemetry', () => { + it('emits failure telemetry for non-existent memory', async () => { + const result = await runCLI(['remove', 'memory', '--name', 'DoesNotExist', '--json'], project.projectPath, { + env: telemetry.env, + }); + + expect(result.exitCode).toBe(1); + telemetry.assertMetricEmitted({ command: 'remove.memory', exit_reason: 'failure' }); + }); + + it('emits failure telemetry for non-existent credential', async () => { + const result = await runCLI(['remove', 'credential', '--name', 'DoesNotExist', '--json'], project.projectPath, { + env: telemetry.env, + }); + + expect(result.exitCode).toBe(1); + telemetry.assertMetricEmitted({ command: 'remove.credential', exit_reason: 'failure' }); + }); + }); + + describe('remove all', () => { + it('resets all schemas and emits telemetry', async () => { + const result = await runCLI(['remove', 'all', '--yes', '--json'], project.projectPath, { + env: telemetry.env, + }); + + expect(result.exitCode).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + telemetry.assertMetricEmitted({ command: 'remove.all', exit_reason: 'success' }); }); }); }); diff --git a/src/cli/commands/remove/command.tsx b/src/cli/commands/remove/command.tsx index 05e532688..da1951d19 100644 --- a/src/cli/commands/remove/command.tsx +++ b/src/cli/commands/remove/command.tsx @@ -1,5 +1,6 @@ import { ConfigIO } from '../../../lib'; import { getErrorMessage } from '../../errors'; +import { runCliCommand } from '../../telemetry/cli-command-run.js'; import { COMMAND_DESCRIPTIONS } from '../../tui/copy'; import { requireProject, requireTTY } from '../../tui/guards'; import { RemoveAllScreen, RemoveFlow } from '../../tui/screens/remove'; @@ -54,9 +55,12 @@ async function handleRemoveAll(_options: RemoveAllOptions): Promise { validateRemoveAllOptions(options); - const result = await handleRemoveAll(options); - console.log(JSON.stringify(result)); - process.exit(result.success ? 0 : 1); + await runCliCommand('remove.all', !!options.json, async () => { + const result = await handleRemoveAll(options); + if (!result.success) throw new Error(result.error); + console.log(JSON.stringify(result)); + return {}; + }); } export const registerRemove = (program: Command): Command => { diff --git a/src/cli/primitives/ABTestPrimitive.ts b/src/cli/primitives/ABTestPrimitive.ts index 9dd973571..dc1852c1f 100644 --- a/src/cli/primitives/ABTestPrimitive.ts +++ b/src/cli/primitives/ABTestPrimitive.ts @@ -3,6 +3,7 @@ import type { ABTest } from '../../schema/schemas/primitives/ab-test'; import { ABTestSchema } from '../../schema/schemas/primitives/ab-test'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; +import { withCommandRunTelemetry } from '../telemetry/cli-command-run.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; import type { AddResult, AddScreenComponent, RemovableResource } from './types'; @@ -468,7 +469,9 @@ Target-Based Mode (--mode target-based) process.exit(1); } - const result = await this.remove(cliOptions.name, { deleteGateway: cliOptions.deleteGateway }); + const result = await withCommandRunTelemetry('remove.ab-test', {}, () => + this.remove(cliOptions.name!, { deleteGateway: cliOptions.deleteGateway }) + ); console.log( JSON.stringify({ success: result.success, diff --git a/src/cli/primitives/AgentPrimitive.tsx b/src/cli/primitives/AgentPrimitive.tsx index b9873990b..283e91ce2 100644 --- a/src/cli/primitives/AgentPrimitive.tsx +++ b/src/cli/primitives/AgentPrimitive.tsx @@ -35,7 +35,7 @@ import { import { executeImportAgent } from '../operations/agent/import'; import { setupPythonProject } from '../operations/python'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { runCliCommand } from '../telemetry/cli-command-run.js'; import { AgentType, AuthorizerType, @@ -283,7 +283,7 @@ export class AgentPrimitive extends BasePrimitive { + await runCliCommand('add.agent', !!cliOptions.json, async () => { const validation = validateAddAgentOptions(cliOptions); if (!validation.valid) { throw new Error(validation.error); diff --git a/src/cli/primitives/BasePrimitive.ts b/src/cli/primitives/BasePrimitive.ts index 149611c4f..4738a7140 100644 --- a/src/cli/primitives/BasePrimitive.ts +++ b/src/cli/primitives/BasePrimitive.ts @@ -2,6 +2,8 @@ import { ConfigIO, findConfigRoot } from '../../lib'; import type { AgentCoreProjectSpec } from '../../schema'; import type { ResourceType } from '../commands/remove/types'; import { getErrorMessage } from '../errors'; +import { withCommandRunTelemetry } from '../telemetry/cli-command-run.js'; +import type { SubCommand } from '../telemetry/schemas/command-run.js'; import { requireTTY } from '../tui/guards/tty'; import { SOURCE_CODE_NOTE } from './constants'; import type { AddResult, AddScreenComponent, RemovableResource, RemovalPreview, RemovalResult } from './types'; @@ -120,7 +122,11 @@ export abstract class BasePrimitive< process.exit(1); } - const result = await this.remove(cliOptions.name); + const result = await withCommandRunTelemetry, RemovalResult>( + `remove.${this.kind}`, + {}, + () => this.remove(cliOptions.name!) + ); console.log( JSON.stringify({ success: result.success, diff --git a/src/cli/primitives/CredentialPrimitive.tsx b/src/cli/primitives/CredentialPrimitive.tsx index 9607094f8..17d77c458 100644 --- a/src/cli/primitives/CredentialPrimitive.tsx +++ b/src/cli/primitives/CredentialPrimitive.tsx @@ -4,7 +4,7 @@ import { CredentialSchema } from '../../schema'; import { validateAddCredentialOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { runCliCommand } from '../telemetry/cli-command-run.js'; import { CredentialType, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; @@ -291,7 +291,7 @@ export class CredentialPrimitive extends BasePrimitive { + await runCliCommand('add.credential', !!cliOptions.json, async () => { const validation = validateAddCredentialOptions({ name: cliOptions.name, type: cliOptions.type as 'api-key' | 'oauth' | undefined, diff --git a/src/cli/primitives/EvaluatorPrimitive.ts b/src/cli/primitives/EvaluatorPrimitive.ts index ec7372b08..ed255c061 100644 --- a/src/cli/primitives/EvaluatorPrimitive.ts +++ b/src/cli/primitives/EvaluatorPrimitive.ts @@ -3,7 +3,7 @@ import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema'; import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { runCliCommand } from '../telemetry/cli-command-run.js'; import { EvaluatorType, Level, standardize } from '../telemetry/schemas/common-shapes.js'; import { renderCodeBasedEvaluatorTemplate } from '../templates/EvaluatorRenderer'; import { requireTTY } from '../tui/guards/tty'; @@ -204,7 +204,7 @@ export class EvaluatorPrimitive extends BasePrimitive { + await runCliCommand('add.evaluator', !!cliOptions.json, async () => { const fail = (error: string): never => { throw new Error(error); }; diff --git a/src/cli/primitives/GatewayPrimitive.ts b/src/cli/primitives/GatewayPrimitive.ts index 625a3c4a5..9710ec076 100644 --- a/src/cli/primitives/GatewayPrimitive.ts +++ b/src/cli/primitives/GatewayPrimitive.ts @@ -12,7 +12,7 @@ import type { AddGatewayOptions as CLIAddGatewayOptions } from '../commands/add/ import { validateAddGatewayOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { runCliCommand, withCommandRunTelemetry } from '../telemetry/cli-command-run.js'; import { AuthorizerType, PolicyEngineMode, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import type { AddGatewayConfig } from '../tui/screens/mcp/types'; @@ -188,7 +188,7 @@ export class GatewayPrimitive extends BasePrimitive { + await runCliCommand('add.gateway', !!cliOptions.json, async () => { const validation = validateAddGatewayOptions(cliOptions); if (!validation.valid) { throw new Error(validation.error); @@ -262,7 +262,7 @@ export class GatewayPrimitive extends BasePrimitive this.remove(cliOptions.name!)); console.log( JSON.stringify({ success: result.success, diff --git a/src/cli/primitives/GatewayTargetPrimitive.ts b/src/cli/primitives/GatewayTargetPrimitive.ts index e8a1da996..2a33b011d 100644 --- a/src/cli/primitives/GatewayTargetPrimitive.ts +++ b/src/cli/primitives/GatewayTargetPrimitive.ts @@ -14,7 +14,7 @@ import { validateAddGatewayTargetOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovableGatewayTarget } from '../operations/remove/remove-gateway-target'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { runCliCommand, withCommandRunTelemetry } from '../telemetry/cli-command-run.js'; import { GATEWAY_TARGET_TYPE_MAP, GatewayTargetHost, @@ -309,7 +309,7 @@ export class GatewayTargetPrimitive extends BasePrimitive { + await runCliCommand('add.gateway-target', !!cliOptions.json, async () => { const validation = await validateAddGatewayTargetOptions(cliOptions); if (!validation.valid) { throw new Error(validation.error); @@ -510,7 +510,9 @@ export class GatewayTargetPrimitive extends BasePrimitive + this.remove(cliOptions.name!) + ); console.log( JSON.stringify({ success: result.success, diff --git a/src/cli/primitives/MemoryPrimitive.tsx b/src/cli/primitives/MemoryPrimitive.tsx index a0c15c2c3..fd893a36e 100644 --- a/src/cli/primitives/MemoryPrimitive.tsx +++ b/src/cli/primitives/MemoryPrimitive.tsx @@ -17,7 +17,7 @@ import { import { DEFAULT_DELIVERY_TYPE, validateAddMemoryOptions } from '../commands/add/validate'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { runCliCommand } from '../telemetry/cli-command-run.js'; import { requireTTY } from '../tui/guards/tty'; import { DEFAULT_EVENT_EXPIRY } from '../tui/screens/memory/types'; import { BasePrimitive } from './BasePrimitive'; @@ -191,7 +191,7 @@ export class MemoryPrimitive extends BasePrimitive { + await runCliCommand('add.memory', !!cliOptions.json, async () => { const expiry = cliOptions.expiry ? parseInt(cliOptions.expiry, 10) : undefined; const validation = validateAddMemoryOptions({ name: cliOptions.name, diff --git a/src/cli/primitives/OnlineEvalConfigPrimitive.ts b/src/cli/primitives/OnlineEvalConfigPrimitive.ts index 6e4436ac9..0b77cbba5 100644 --- a/src/cli/primitives/OnlineEvalConfigPrimitive.ts +++ b/src/cli/primitives/OnlineEvalConfigPrimitive.ts @@ -3,7 +3,7 @@ import type { OnlineEvalConfig } from '../../schema'; import { OnlineEvalConfigSchema } from '../../schema'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { runCliCommand } from '../telemetry/cli-command-run.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; import type { AddResult, AddScreenComponent, RemovableResource } from './types'; @@ -134,7 +134,7 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive { + await runCliCommand('add.online-eval', !!cliOptions.json, async () => { if (!cliOptions.name || !cliOptions.runtime || allEvaluators.length === 0 || !cliOptions.samplingRate) { throw new Error( '--name, --runtime, --evaluator (and/or --evaluator-arn), and --sampling-rate are all required in non-interactive mode' diff --git a/src/cli/primitives/PolicyEnginePrimitive.ts b/src/cli/primitives/PolicyEnginePrimitive.ts index bb9b314d8..bab91022a 100644 --- a/src/cli/primitives/PolicyEnginePrimitive.ts +++ b/src/cli/primitives/PolicyEnginePrimitive.ts @@ -3,7 +3,7 @@ import type { AgentCoreProjectSpec, PolicyEngine } from '../../schema'; import { PolicyEngineModeSchema, PolicyEngineSchema } from '../../schema'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { runCliCommand, withCommandRunTelemetry } from '../telemetry/cli-command-run.js'; import { AttachMode, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; @@ -229,7 +229,7 @@ export class PolicyEnginePrimitive extends BasePrimitive { + await runCliCommand('add.policy-engine', !!cliOptions.json, async () => { if (!cliOptions.name) { throw new Error('--name is required'); } @@ -316,7 +316,9 @@ export class PolicyEnginePrimitive extends BasePrimitive + this.remove(cliOptions.name!) + ); if (cliOptions.json) { console.log( JSON.stringify({ diff --git a/src/cli/primitives/PolicyPrimitive.ts b/src/cli/primitives/PolicyPrimitive.ts index beefe3008..e9d3b6914 100644 --- a/src/cli/primitives/PolicyPrimitive.ts +++ b/src/cli/primitives/PolicyPrimitive.ts @@ -5,7 +5,7 @@ import { detectRegion } from '../aws'; import { getPolicyGeneration, startPolicyGeneration } from '../aws/policy-generation'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { runCliCommand, withCommandRunTelemetry } from '../telemetry/cli-command-run.js'; import { ValidationMode, standardize } from '../telemetry/schemas/common-shapes.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; @@ -306,7 +306,7 @@ export class PolicyPrimitive extends BasePrimitive { + await runCliCommand('add.policy', !!cliOptions.json, async () => { if (!cliOptions.name) { throw new Error('--name is required'); } @@ -400,7 +400,7 @@ export class PolicyPrimitive extends BasePrimitive this.remove(removeKey)); if (cliOptions.json) { console.log( diff --git a/src/cli/primitives/RuntimeEndpointPrimitive.ts b/src/cli/primitives/RuntimeEndpointPrimitive.ts index e055fd17a..ae2ae83b1 100644 --- a/src/cli/primitives/RuntimeEndpointPrimitive.ts +++ b/src/cli/primitives/RuntimeEndpointPrimitive.ts @@ -4,7 +4,7 @@ import { RuntimeEndpointSchema } from '../../schema'; import type { ResourceType } from '../commands/remove/types'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; -import { cliCommandRun } from '../telemetry/cli-command-run.js'; +import { runCliCommand, withCommandRunTelemetry } from '../telemetry/cli-command-run.js'; import { BasePrimitive } from './BasePrimitive'; import { SOURCE_CODE_NOTE } from './constants'; import type { AddResult, AddScreenComponent, RemovableResource } from './types'; @@ -254,7 +254,7 @@ export class RuntimeEndpointPrimitive extends BasePrimitive { + await runCliCommand('add.runtime-endpoint', !!cliOptions.json, async () => { const result = await this.add({ runtime: cliOptions.runtime, endpoint: cliOptions.endpoint, @@ -296,7 +296,9 @@ export class RuntimeEndpointPrimitive extends BasePrimitive + this.remove(cliOptions.name!) + ); console.log( JSON.stringify({ success: result.success, diff --git a/src/cli/telemetry/cli-command-run.ts b/src/cli/telemetry/cli-command-run.ts index 987f05730..f04347dc5 100644 --- a/src/cli/telemetry/cli-command-run.ts +++ b/src/cli/telemetry/cli-command-run.ts @@ -1,30 +1,69 @@ import { getErrorMessage } from '../errors'; -import type { AddResult } from '../primitives/types.js'; import { TelemetryClientAccessor } from './client-accessor.js'; import type { Command, CommandAttrs } from './schemas/command-run.js'; +// TODO: Replace with a generic Result type that preserves the original error object. +export type OperationResult = { success: true } | { success: false; error: string }; + +async function getTelemetryClient() { + try { + return await TelemetryClientAccessor.get(); + } catch { + return undefined; + } +} + /** - * Run a CLI command with telemetry, standardized error output, and process.exit. - * The callback should throw on failure and return telemetry attrs on success. + * Record telemetry for an operation and return its result. + * Use in TUI hooks and CLI paths where the caller handles output and control flow. * - * If telemetry initialization fails, the command still runs without telemetry — - * telemetry must never block CLI behavior. + * If the callback returns a failure result, telemetry is recorded and the result + * is returned to the caller. If the callback throws, telemetry is recorded and + * the exception propagates. If telemetry is unavailable, the callback runs untracked. */ -export async function cliCommandRun( +export async function withCommandRunTelemetry( + command: C, + attrs: CommandAttrs, + fn: () => Promise +): Promise { + const client = await getTelemetryClient(); + if (!client) return fn(); + + let result: R | undefined; + try { + await client.withCommandRun(command, async () => { + result = await fn(); + if (!result.success) throw new Error(result.error); + return attrs; + }); + } catch (e) { + // withCommandRun re-throws after recording failure telemetry. + // If result was set, fn() returned a failure result — return it directly. + // If not, fn() itself threw — convert to a failure result so callers + // that don't wrap in try/catch (e.g. TUI hooks) don't leak unhandled rejections. + if (!result) { + return { success: false, error: getErrorMessage(e) } as R; + } + } + return result!; +} + +/** + * Record telemetry, print errors, and exit the process. + * Use in CLI command handlers where the command is the final action. + * The callback returns attrs on success and throws on failure. + */ +export async function runCliCommand( command: C, json: boolean, fn: () => Promise> ): Promise { try { - let client; - try { - client = await TelemetryClientAccessor.get(); - } catch { - // Telemetry init failed — run without it + const client = await getTelemetryClient(); + if (!client) { await fn(); process.exit(0); } - // withCommandRun records success/failure telemetry, then re-throws on failure await client.withCommandRun(command, fn); process.exit(0); } catch (error) { @@ -36,36 +75,3 @@ export async function cliCommandRun( process.exit(1); } } - -/** - * Wrap a primitive .add() call with telemetry — used by TUI paths. - * CLI paths use {@link cliCommandRun} instead. - */ -export async function withAddTelemetry>( - command: C, - attrs: CommandAttrs, - fn: () => Promise> -): Promise> { - let client; - try { - client = await TelemetryClientAccessor.get(); - } catch { - return fn(); - } - - let result: AddResult | undefined; - try { - await client.withCommandRun(command, async () => { - result = await fn(); - if (!result.success) throw new Error(result.error); - return attrs; - }); - } catch (err) { - // withCommandRun re-throws after recording failure telemetry. - // result is set if fn() ran; if not, fn() itself threw. - if (!result) { - return { success: false, error: getErrorMessage(err) }; - } - } - return result!; -} diff --git a/src/cli/telemetry/schemas/command-run.ts b/src/cli/telemetry/schemas/command-run.ts index 0acfcaf1b..7bcc4e555 100644 --- a/src/cli/telemetry/schemas/command-run.ts +++ b/src/cli/telemetry/schemas/command-run.ts @@ -195,6 +195,7 @@ export const COMMAND_SCHEMAS = { validate: NoAttrs, 'help.modes': NoAttrs, help: NoAttrs, + 'remove.all': NoAttrs, 'remove.agent': NoAttrs, 'remove.memory': NoAttrs, 'remove.credential': NoAttrs, @@ -204,6 +205,9 @@ export const COMMAND_SCHEMAS = { 'remove.gateway-target': NoAttrs, 'remove.policy-engine': NoAttrs, 'remove.policy': NoAttrs, + 'remove.runtime-endpoint': NoAttrs, + 'remove.config-bundle': NoAttrs, + 'remove.ab-test': NoAttrs, 'telemetry.disable': NoAttrs, 'telemetry.enable': NoAttrs, 'telemetry.status': NoAttrs, @@ -216,6 +220,22 @@ export const COMMAND_SCHEMAS = { export type Command = keyof typeof COMMAND_SCHEMAS; export type CommandAttrs = z.infer<(typeof COMMAND_SCHEMAS)[C]>; +/** Extract the command group prefix from a dotted command key (e.g. 'add' from 'add.agent'). */ +type CommandGroup = { + [C in Command]: C extends `${infer G}.${string}` ? G : C; +}[Command]; + +/** + * Type-safe lookup of a subcommand under a command group. + * Produces a compile-time error if `${G}.${S}` is not a registered command. + * + * @example + * SubCommand<'remove', 'agent'> // → 'remove.agent' + * SubCommand<'add', 'memory'> // → 'add.memory' + * SubCommand<'remove', 'bogus'> // → never (compile error at call site) + */ +export type SubCommand = Extract; + /** Derive command_group from command key (e.g. 'add.agent' → 'add') */ export function deriveCommandGroup(command: Command): string { const dot = command.indexOf('.'); diff --git a/src/cli/tui/hooks/useCreateEvaluator.ts b/src/cli/tui/hooks/useCreateEvaluator.ts index f1cad666f..087eca479 100644 --- a/src/cli/tui/hooks/useCreateEvaluator.ts +++ b/src/cli/tui/hooks/useCreateEvaluator.ts @@ -1,6 +1,6 @@ import type { EvaluatorConfig } from '../../../schema'; import { evaluatorPrimitive } from '../../primitives/registry'; -import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; +import { withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; import { Level, standardize } from '../../telemetry/schemas/common-shapes.js'; import { useCallback, useEffect, useState } from 'react'; @@ -18,7 +18,7 @@ export function useCreateEvaluator() { const create = useCallback(async (config: CreateEvaluatorConfig) => { setStatus({ state: 'loading' }); try { - const addResult = await withAddTelemetry( + const addResult = await withCommandRunTelemetry( 'add.evaluator', { evaluator_type: config.config.codeBased ? 'code-based' : 'llm-as-a-judge', diff --git a/src/cli/tui/hooks/useCreateMcp.ts b/src/cli/tui/hooks/useCreateMcp.ts index ec91666d0..60900bc63 100644 --- a/src/cli/tui/hooks/useCreateMcp.ts +++ b/src/cli/tui/hooks/useCreateMcp.ts @@ -4,7 +4,7 @@ import { gatewayTargetPrimitive, policyEnginePrimitive, } from '../../primitives/registry'; -import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; +import { withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; import { AuthorizerType, PolicyEngineMode, standardize } from '../../telemetry/schemas/common-shapes.js'; import type { AddGatewayConfig } from '../screens/mcp/types'; import { useCallback, useEffect, useState } from 'react'; @@ -25,7 +25,7 @@ export function useCreateGateway() { const createGateway = useCallback(async (config: AddGatewayConfig) => { setStatus({ state: 'loading' }); try { - const addResult = await withAddTelemetry( + const addResult = await withCommandRunTelemetry( 'add.gateway', { authorizer_type: standardize(AuthorizerType, config.authorizerType ?? 'NONE'), diff --git a/src/cli/tui/hooks/useCreateMemory.ts b/src/cli/tui/hooks/useCreateMemory.ts index d4196582f..f125b2615 100644 --- a/src/cli/tui/hooks/useCreateMemory.ts +++ b/src/cli/tui/hooks/useCreateMemory.ts @@ -2,7 +2,7 @@ import { ConfigIO } from '../../../lib'; import type { Memory } from '../../../schema'; import { getAvailableAgents } from '../../operations/attach'; import { memoryPrimitive } from '../../primitives/registry'; -import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; +import { withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; import { useCallback, useEffect, useState } from 'react'; interface CreateMemoryConfig { @@ -26,7 +26,7 @@ export function useCreateMemory() { try { const strategiesStr = config.strategies.map(s => s.type).join(','); const strategyList = strategiesStr ? strategiesStr.split(',').map(s => s.trim().toUpperCase()) : []; - const addResult = await withAddTelemetry( + const addResult = await withCommandRunTelemetry( 'add.memory', { strategy_count: strategyList.length, diff --git a/src/cli/tui/hooks/useCreateOnlineEval.ts b/src/cli/tui/hooks/useCreateOnlineEval.ts index b853fed05..381e6d136 100644 --- a/src/cli/tui/hooks/useCreateOnlineEval.ts +++ b/src/cli/tui/hooks/useCreateOnlineEval.ts @@ -1,5 +1,5 @@ import { onlineEvalConfigPrimitive } from '../../primitives/registry'; -import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; +import { withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; import { useCallback, useEffect, useState } from 'react'; interface CreateOnlineEvalConfig { @@ -20,7 +20,7 @@ export function useCreateOnlineEval() { const create = useCallback(async (config: CreateOnlineEvalConfig) => { setStatus({ state: 'loading' }); try { - const addResult = await withAddTelemetry( + const addResult = await withCommandRunTelemetry( 'add.online-eval', { evaluator_count: config.evaluators.length, diff --git a/src/cli/tui/hooks/useRemove.ts b/src/cli/tui/hooks/useRemove.ts index 7682479d3..abdeda2d9 100644 --- a/src/cli/tui/hooks/useRemove.ts +++ b/src/cli/tui/hooks/useRemove.ts @@ -19,6 +19,8 @@ import { policyPrimitive, runtimeEndpointPrimitive, } from '../../primitives/registry'; +import { withCommandRunTelemetry } from '../../telemetry/cli-command-run.js'; +import type { SubCommand } from '../../telemetry/schemas/command-run.js'; import { useCallback, useEffect, useRef, useState } from 'react'; // Re-export types for consumers @@ -74,7 +76,11 @@ function useRemoveResource( const remove = useCallback(async (id: TIdentifier, preview?: RemovalPreview): Promise => { setState({ isLoading: true, result: null }); - const result = await removeFnRef.current(id); + const result = await withCommandRunTelemetry, RemovalResult>( + `remove.${resourceTypeRef.current}`, + {}, + () => removeFnRef.current(id) + ); setState({ isLoading: false, result }); let logPath: string | undefined; diff --git a/src/cli/tui/screens/agent/useAddAgent.ts b/src/cli/tui/screens/agent/useAddAgent.ts index e2fabe140..aeac9e442 100644 --- a/src/cli/tui/screens/agent/useAddAgent.ts +++ b/src/cli/tui/screens/agent/useAddAgent.ts @@ -12,7 +12,7 @@ import { executeImportAgent } from '../../../operations/agent/import'; import { buildAuthorizerConfigFromJwtConfig, createManagedOAuthCredential } from '../../../primitives/auth-utils'; import { computeDefaultCredentialEnvVarName } from '../../../primitives/credential-utils'; import { credentialPrimitive } from '../../../primitives/registry'; -import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; +import { withCommandRunTelemetry } from '../../../telemetry/cli-command-run.js'; import { AgentType as AgentTypeEnum, AuthorizerType as AuthorizerTypeEnum, @@ -149,7 +149,7 @@ export function useAddAgent() { const addAgent = useCallback(async (config: AddAgentConfig): Promise => { setIsLoading(true); try { - const result = await withAddTelemetry( + const result = await withCommandRunTelemetry( 'add.agent', { language: standardize(Language, config.language), diff --git a/src/cli/tui/screens/identity/useCreateIdentity.ts b/src/cli/tui/screens/identity/useCreateIdentity.ts index e214d1e67..c9293665b 100644 --- a/src/cli/tui/screens/identity/useCreateIdentity.ts +++ b/src/cli/tui/screens/identity/useCreateIdentity.ts @@ -2,7 +2,7 @@ import { ConfigIO } from '../../../../lib'; import type { Credential } from '../../../../schema'; import type { AddCredentialOptions } from '../../../primitives/CredentialPrimitive'; import { credentialPrimitive } from '../../../primitives/registry'; -import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; +import { withCommandRunTelemetry } from '../../../telemetry/cli-command-run.js'; import { useCallback, useEffect, useState } from 'react'; interface CreateStatus { @@ -17,7 +17,7 @@ export function useCreateIdentity() { const create = useCallback(async (config: AddCredentialOptions) => { setStatus({ state: 'loading' }); try { - const result = await withAddTelemetry( + const result = await withCommandRunTelemetry( 'add.credential', { credential_type: config.authorizerType === 'OAuthCredentialProvider' ? 'oauth' : 'api-key', diff --git a/src/cli/tui/screens/policy/AddPolicyFlow.tsx b/src/cli/tui/screens/policy/AddPolicyFlow.tsx index 9b3542cb8..8238d84c8 100644 --- a/src/cli/tui/screens/policy/AddPolicyFlow.tsx +++ b/src/cli/tui/screens/policy/AddPolicyFlow.tsx @@ -1,5 +1,5 @@ import { policyEnginePrimitive, policyPrimitive } from '../../../primitives/registry'; -import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; +import { withCommandRunTelemetry } from '../../../telemetry/cli-command-run.js'; import { AttachMode, ValidationMode, standardize } from '../../../telemetry/schemas/common-shapes.js'; import { ErrorPrompt, @@ -130,7 +130,7 @@ export function AddPolicyFlow({ isInteractive = true, onExit, onBack, onDev, onD }, []); const commitEngine = useCallback(async (engineName: string, gateways?: string[], mode?: 'LOG_ONLY' | 'ENFORCE') => { - const result = await withAddTelemetry( + const result = await withCommandRunTelemetry( 'add.policy-engine', { attach_gateway_count: gateways?.length ?? 0, @@ -164,7 +164,7 @@ export function AddPolicyFlow({ isInteractive = true, onExit, onBack, onDev, onD ); const handlePolicyComplete = useCallback(async (config: AddPolicyConfig) => { - const result = await withAddTelemetry( + const result = await withCommandRunTelemetry( 'add.policy', { source_type: config.sourceFile ? 'file' : config.sourceMethod === 'generate' ? 'generate' : 'statement', diff --git a/src/cli/tui/screens/remove/useRemoveFlow.ts b/src/cli/tui/screens/remove/useRemoveFlow.ts index 8c4d3ef69..96ddd2f64 100644 --- a/src/cli/tui/screens/remove/useRemoveFlow.ts +++ b/src/cli/tui/screens/remove/useRemoveFlow.ts @@ -1,7 +1,9 @@ import { ConfigIO, getWorkingDirectory } from '../../../../lib'; import { findStack } from '../../../cloudformation/stack-discovery'; import { getErrorMessage } from '../../../errors'; +import type { RemovalResult } from '../../../operations/remove/types'; import { createDefaultProjectSpec } from '../../../project'; +import { withCommandRunTelemetry } from '../../../telemetry/cli-command-run.js'; import { type Step, areStepsComplete, hasStepError } from '../../components'; import { withMinDuration } from '../../utils'; import { useCallback, useEffect, useMemo, useState } from 'react'; @@ -146,16 +148,23 @@ export function useRemoveFlow({ force, dryRun }: RemoveFlowOptions): RemoveFlowS // Reset all schemas to default empty state updateStep(0, { status: 'running' }); try { - await withMinDuration(async () => { - const configIO = new ConfigIO(); - - // Reset agentcore.json (keep project name) - const defaultProjectSpec = createDefaultProjectSpec(projectName || 'Project'); - await configIO.writeProjectSpec(defaultProjectSpec); - - // Preserve aws-targets.json and deployed-state.json so that - // a subsequent `agentcore deploy` can tear down existing stacks. - }); + const res = await withCommandRunTelemetry( + 'remove.all', + {}, + (): Promise => + withMinDuration(async () => { + const configIO = new ConfigIO(); + + // Reset agentcore.json (keep project name) + const defaultProjectSpec = createDefaultProjectSpec(projectName || 'Project'); + await configIO.writeProjectSpec(defaultProjectSpec); + + // Preserve aws-targets.json and deployed-state.json so that + // a subsequent `agentcore deploy` can tear down existing stacks. + return { success: true }; + }) + ); + if (!res.success) throw new Error(res.error); updateStep(0, { status: 'success' }); } catch (err) { updateStep(0, { status: 'error', error: getErrorMessage(err) }); diff --git a/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx b/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx index 83bda78e5..77c07b2e8 100644 --- a/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx +++ b/src/cli/tui/screens/runtime-endpoint/AddRuntimeEndpointFlow.tsx @@ -1,6 +1,6 @@ import { ConfigIO } from '../../../../lib'; import { runtimeEndpointPrimitive } from '../../../primitives/registry'; -import { withAddTelemetry } from '../../../telemetry/cli-command-run.js'; +import { withCommandRunTelemetry } from '../../../telemetry/cli-command-run.js'; import { ErrorPrompt } from '../../components'; import { AddSuccessScreen } from '../add/AddSuccessScreen'; import { AddRuntimeEndpointScreen } from './AddRuntimeEndpointScreen'; @@ -79,7 +79,7 @@ export function AddRuntimeEndpointFlow({ }, [isInteractive, flow.name, onExit]); const handleCreateComplete = useCallback((config: RuntimeEndpointWizardConfig) => { - void withAddTelemetry('add.runtime-endpoint', {}, () => + void withCommandRunTelemetry('add.runtime-endpoint', {}, () => runtimeEndpointPrimitive.add({ runtime: config.runtimeName, endpoint: config.endpointName,