From 8944f6b057e2e8b6c36a48ce75aec38208190be8 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 11 Mar 2026 10:09:31 +0000 Subject: [PATCH 01/30] feat: add automatic LLM cost tracking for GenAI spans Calculates costs from gen_ai.* span attributes using an in-memory pricing registry backed by Postgres, with model prices synced from Langfuse (145 models). Costs are dual-written to span attributes (trigger.llm.*) and a new llm_usage_v1 ClickHouse table for efficient aggregation. - New @internal/llm-pricing package with ModelPricingRegistry - Prisma schema for llm_models, llm_pricing_tiers, llm_prices - ClickHouse llm_usage_v1 table with DynamicFlushScheduler batching - Cost enrichment in enrichCreatableEvents() with gen_ai.usage.* extraction - TRQL llm_usage table schema for querying - Admin API endpoints for model CRUD, seed, and registry reload - Pill-style accessories on spans showing model, tokens, and cost - Anthropic logo icon for RunIcon - Style merge fix for partial/completed span deduplication - Env vars: LLM_COST_TRACKING_ENABLED, LLM_PRICING_RELOAD_INTERVAL_MS refs TRI-7773 --- .server-changes/llm-cost-tracking.md | 6 + .../app/assets/icons/AnthropicLogoIcon.tsx | 12 + .../webapp/app/components/runs/v3/RunIcon.tsx | 3 + .../app/components/runs/v3/SpanTitle.tsx | 26 + apps/webapp/app/env.server.ts | 4 + .../admin.api.v1.llm-models.$modelId.ts | 149 + .../routes/admin.api.v1.llm-models.reload.ts | 24 + .../routes/admin.api.v1.llm-models.seed.ts | 30 + .../app/routes/admin.api.v1.llm-models.ts | 132 + .../clickhouseEventRepository.server.ts | 86 +- .../eventRepository/eventRepository.types.ts | 20 + .../app/v3/llmPricingRegistry.server.ts | 47 + apps/webapp/app/v3/otlpExporter.server.ts | 1 + apps/webapp/app/v3/querySchemas.ts | 146 +- .../v3/utils/enrichCreatableEvents.server.ts | 183 +- apps/webapp/package.json | 1 + apps/webapp/test/otlpExporter.test.ts | 240 +- .../schema/024_create_llm_usage_v1.sql | 45 + internal-packages/clickhouse/src/index.ts | 8 + internal-packages/clickhouse/src/llmUsage.ts | 42 + .../migration.sql | 63 + .../database/prisma/schema.prisma | 56 + internal-packages/llm-pricing/package.json | 17 + .../llm-pricing/scripts/sync-model-prices.sh | 77 + .../llm-pricing/src/default-model-prices.json | 3838 +++++++++++++++++ .../llm-pricing/src/defaultPrices.ts | 2984 +++++++++++++ internal-packages/llm-pricing/src/index.ts | 11 + .../llm-pricing/src/registry.test.ts | 233 + internal-packages/llm-pricing/src/registry.ts | 209 + internal-packages/llm-pricing/src/seed.ts | 59 + internal-packages/llm-pricing/src/types.ts | 53 + internal-packages/llm-pricing/tsconfig.json | 19 + packages/core/src/v3/schemas/style.ts | 3 +- 33 files changed, 8821 insertions(+), 6 deletions(-) create mode 100644 .server-changes/llm-cost-tracking.md create mode 100644 apps/webapp/app/assets/icons/AnthropicLogoIcon.tsx create mode 100644 apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts create mode 100644 apps/webapp/app/routes/admin.api.v1.llm-models.reload.ts create mode 100644 apps/webapp/app/routes/admin.api.v1.llm-models.seed.ts create mode 100644 apps/webapp/app/routes/admin.api.v1.llm-models.ts create mode 100644 apps/webapp/app/v3/llmPricingRegistry.server.ts create mode 100644 internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql create mode 100644 internal-packages/clickhouse/src/llmUsage.ts create mode 100644 internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql create mode 100644 internal-packages/llm-pricing/package.json create mode 100755 internal-packages/llm-pricing/scripts/sync-model-prices.sh create mode 100644 internal-packages/llm-pricing/src/default-model-prices.json create mode 100644 internal-packages/llm-pricing/src/defaultPrices.ts create mode 100644 internal-packages/llm-pricing/src/index.ts create mode 100644 internal-packages/llm-pricing/src/registry.test.ts create mode 100644 internal-packages/llm-pricing/src/registry.ts create mode 100644 internal-packages/llm-pricing/src/seed.ts create mode 100644 internal-packages/llm-pricing/src/types.ts create mode 100644 internal-packages/llm-pricing/tsconfig.json diff --git a/.server-changes/llm-cost-tracking.md b/.server-changes/llm-cost-tracking.md new file mode 100644 index 00000000000..b68302731a7 --- /dev/null +++ b/.server-changes/llm-cost-tracking.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Add automatic LLM cost calculation for spans with GenAI semantic conventions. When a span arrives with `gen_ai.response.model` and token usage data, costs are calculated from an in-memory pricing registry backed by Postgres and dual-written to both span attributes (`trigger.llm.*`) and a new `llm_usage_v1` ClickHouse table. diff --git a/apps/webapp/app/assets/icons/AnthropicLogoIcon.tsx b/apps/webapp/app/assets/icons/AnthropicLogoIcon.tsx new file mode 100644 index 00000000000..3e647284cce --- /dev/null +++ b/apps/webapp/app/assets/icons/AnthropicLogoIcon.tsx @@ -0,0 +1,12 @@ +export function AnthropicLogoIcon({ className }: { className?: string }) { + return ( + + + + ); +} diff --git a/apps/webapp/app/components/runs/v3/RunIcon.tsx b/apps/webapp/app/components/runs/v3/RunIcon.tsx index 846d7cae0a4..73963ea09b9 100644 --- a/apps/webapp/app/components/runs/v3/RunIcon.tsx +++ b/apps/webapp/app/components/runs/v3/RunIcon.tsx @@ -7,6 +7,7 @@ import { TableCellsIcon, TagIcon, } from "@heroicons/react/20/solid"; +import { AnthropicLogoIcon } from "~/assets/icons/AnthropicLogoIcon"; import { AttemptIcon } from "~/assets/icons/AttemptIcon"; import { TaskIcon } from "~/assets/icons/TaskIcon"; import { cn } from "~/utils/cn"; @@ -112,6 +113,8 @@ export function RunIcon({ name, className, spanName }: TaskIconProps) { return ; case "streams": return ; + case "tabler-brand-anthropic": + return ; } return ; diff --git a/apps/webapp/app/components/runs/v3/SpanTitle.tsx b/apps/webapp/app/components/runs/v3/SpanTitle.tsx index fb0105c45db..fe85fd70c9e 100644 --- a/apps/webapp/app/components/runs/v3/SpanTitle.tsx +++ b/apps/webapp/app/components/runs/v3/SpanTitle.tsx @@ -3,6 +3,8 @@ import { TaskEventStyle } from "@trigger.dev/core/v3"; import type { TaskEventLevel } from "@trigger.dev/database"; import { Fragment } from "react"; import { cn } from "~/utils/cn"; +import { tablerIcons } from "~/utils/tablerIcons"; +import tablerSpritePath from "~/components/primitives/tabler-sprite.svg"; type SpanTitleProps = { message: string; @@ -45,6 +47,15 @@ function SpanAccessory({ /> ); } + case "pills": { + return ( +
+ {accessory.items.map((item, index) => ( + + ))} +
+ ); + } default: { return (
@@ -59,6 +70,21 @@ function SpanAccessory({ } } +function SpanPill({ text, icon }: { text: string; icon?: string }) { + const hasIcon = icon && tablerIcons.has(icon); + + return ( + + {hasIcon && ( + + + + )} + {text} + + ); +} + export function SpanCodePathAccessory({ accessory, className, diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index bdfdbea6b3e..8877c982ae0 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1277,6 +1277,10 @@ const EnvironmentSchema = z EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(5_000), EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING: z.coerce.number().int().default(2000), + // LLM cost tracking + LLM_COST_TRACKING_ENABLED: BoolEnv.default(true), + LLM_PRICING_RELOAD_INTERVAL_MS: z.coerce.number().int().default(5 * 60 * 1000), // 5 minutes + // Bootstrap TRIGGER_BOOTSTRAP_ENABLED: z.string().default("0"), TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: z.string().optional(), diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts new file mode 100644 index 00000000000..8cf132daab9 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts @@ -0,0 +1,149 @@ +import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; + +async function requireAdmin(request: Request) { + const authResult = await authenticateApiRequestWithPersonalAccessToken(request); + if (!authResult) { + throw json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ where: { id: authResult.userId } }); + if (!user?.admin) { + throw json({ error: "You must be an admin to perform this action" }, { status: 403 }); + } + + return user; +} + +export async function loader({ request, params }: LoaderFunctionArgs) { + await requireAdmin(request); + + const model = await prisma.llmModel.findUnique({ + where: { id: params.modelId }, + include: { + pricingTiers: { + include: { prices: true }, + orderBy: { priority: "asc" }, + }, + }, + }); + + if (!model) { + return json({ error: "Model not found" }, { status: 404 }); + } + + return json({ model }); +} + +const UpdateModelSchema = z.object({ + modelName: z.string().min(1).optional(), + matchPattern: z.string().min(1).optional(), + startDate: z.string().nullable().optional(), + pricingTiers: z + .array( + z.object({ + name: z.string().min(1), + isDefault: z.boolean().default(true), + priority: z.number().int().default(0), + conditions: z + .array( + z.object({ + usageDetailPattern: z.string(), + operator: z.enum(["gt", "gte", "lt", "lte", "eq", "neq"]), + value: z.number(), + }) + ) + .default([]), + prices: z.record(z.string(), z.number()), + }) + ) + .optional(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + await requireAdmin(request); + + const modelId = params.modelId!; + + if (request.method === "DELETE") { + const existing = await prisma.llmModel.findUnique({ where: { id: modelId } }); + if (!existing) { + return json({ error: "Model not found" }, { status: 404 }); + } + + await prisma.llmModel.delete({ where: { id: modelId } }); + return json({ success: true }); + } + + if (request.method !== "PUT") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const body = await request.json(); + const parsed = UpdateModelSchema.safeParse(body); + + if (!parsed.success) { + return json({ error: "Invalid request body", details: parsed.error.issues }, { status: 400 }); + } + + const { modelName, matchPattern, startDate, pricingTiers } = parsed.data; + + // Validate regex if provided + if (matchPattern) { + try { + new RegExp(matchPattern); + } catch { + return json({ error: "Invalid regex in matchPattern" }, { status: 400 }); + } + } + + // Update model fields + const model = await prisma.llmModel.update({ + where: { id: modelId }, + data: { + ...(modelName !== undefined && { modelName }), + ...(matchPattern !== undefined && { matchPattern }), + ...(startDate !== undefined && { startDate: startDate ? new Date(startDate) : null }), + }, + }); + + // If pricing tiers provided, replace them entirely + if (pricingTiers) { + // Delete existing tiers (cascades to prices) + await prisma.llmPricingTier.deleteMany({ where: { modelId } }); + + // Create new tiers + for (const tier of pricingTiers) { + await prisma.llmPricingTier.create({ + data: { + modelId, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId, + usageType, + price, + })), + }, + }, + }); + } + } + + const updated = await prisma.llmModel.findUnique({ + where: { id: modelId }, + include: { + pricingTiers: { + include: { prices: true }, + orderBy: { priority: "asc" }, + }, + }, + }); + + return json({ model: updated }); +} diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.reload.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.reload.ts new file mode 100644 index 00000000000..747722b352a --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.reload.ts @@ -0,0 +1,24 @@ +import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { prisma } from "~/db.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +export async function action({ request }: ActionFunctionArgs) { + const authResult = await authenticateApiRequestWithPersonalAccessToken(request); + if (!authResult) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ where: { id: authResult.userId } }); + if (!user?.admin) { + return json({ error: "You must be an admin to perform this action" }, { status: 403 }); + } + + if (!llmPricingRegistry) { + return json({ error: "LLM cost tracking is disabled" }, { status: 400 }); + } + + await llmPricingRegistry.reload(); + + return json({ success: true, message: "LLM pricing registry reloaded" }); +} diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.seed.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.seed.ts new file mode 100644 index 00000000000..805f97ad233 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.seed.ts @@ -0,0 +1,30 @@ +import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { seedLlmPricing } from "@internal/llm-pricing"; +import { prisma } from "~/db.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +export async function action({ request }: ActionFunctionArgs) { + const authResult = await authenticateApiRequestWithPersonalAccessToken(request); + if (!authResult) { + return json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ where: { id: authResult.userId } }); + if (!user?.admin) { + return json({ error: "You must be an admin to perform this action" }, { status: 403 }); + } + + const result = await seedLlmPricing(prisma); + + // Reload the in-memory registry after seeding (if enabled) + if (llmPricingRegistry) { + await llmPricingRegistry.reload(); + } + + return json({ + success: true, + ...result, + message: `Seeded ${result.modelsCreated} models, skipped ${result.modelsSkipped} existing`, + }); +} diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.ts new file mode 100644 index 00000000000..7b1b2226ca5 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.ts @@ -0,0 +1,132 @@ +import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; + +async function requireAdmin(request: Request) { + const authResult = await authenticateApiRequestWithPersonalAccessToken(request); + if (!authResult) { + throw json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ where: { id: authResult.userId } }); + if (!user?.admin) { + throw json({ error: "You must be an admin to perform this action" }, { status: 403 }); + } + + return user; +} + +export async function loader({ request }: LoaderFunctionArgs) { + await requireAdmin(request); + + const url = new URL(request.url); + const page = parseInt(url.searchParams.get("page") ?? "1"); + const pageSize = parseInt(url.searchParams.get("pageSize") ?? "50"); + + const [models, total] = await Promise.all([ + prisma.llmModel.findMany({ + where: { projectId: null }, + include: { + pricingTiers: { + include: { prices: true }, + orderBy: { priority: "asc" }, + }, + }, + orderBy: { modelName: "asc" }, + skip: (page - 1) * pageSize, + take: pageSize, + }), + prisma.llmModel.count({ where: { projectId: null } }), + ]); + + return json({ models, total, page, pageSize }); +} + +const CreateModelSchema = z.object({ + modelName: z.string().min(1), + matchPattern: z.string().min(1), + startDate: z.string().optional(), + source: z.enum(["default", "admin"]).optional().default("admin"), + pricingTiers: z.array( + z.object({ + name: z.string().min(1), + isDefault: z.boolean().default(true), + priority: z.number().int().default(0), + conditions: z + .array( + z.object({ + usageDetailPattern: z.string(), + operator: z.enum(["gt", "gte", "lt", "lte", "eq", "neq"]), + value: z.number(), + }) + ) + .default([]), + prices: z.record(z.string(), z.number()), + }) + ), +}); + +export async function action({ request }: ActionFunctionArgs) { + await requireAdmin(request); + + if (request.method !== "POST") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const body = await request.json(); + const parsed = CreateModelSchema.safeParse(body); + + if (!parsed.success) { + return json({ error: "Invalid request body", details: parsed.error.issues }, { status: 400 }); + } + + const { modelName, matchPattern, startDate, source, pricingTiers } = parsed.data; + + // Validate regex pattern + try { + new RegExp(matchPattern); + } catch { + return json({ error: "Invalid regex in matchPattern" }, { status: 400 }); + } + + // Create model first, then tiers with explicit model connection + const model = await prisma.llmModel.create({ + data: { + modelName, + matchPattern, + startDate: startDate ? new Date(startDate) : null, + source, + }, + }); + + for (const tier of pricingTiers) { + await prisma.llmPricingTier.create({ + data: { + modelId: model.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId: model.id, + usageType, + price, + })), + }, + }, + }); + } + + const created = await prisma.llmModel.findUnique({ + where: { id: model.id }, + include: { + pricingTiers: { + include: { prices: true }, + }, + }, + }); + + return json({ model: created }, { status: 201 }); +} diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts index 9100ad84fec..780b74d85e1 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts @@ -1,5 +1,6 @@ import type { ClickHouse, + LlmUsageV1Input, TaskEventDetailedSummaryV1Result, TaskEventDetailsV1Result, TaskEventSummaryV1Result, @@ -7,6 +8,9 @@ import type { TaskEventV2Input, } from "@internal/clickhouse"; import { Attributes, startSpan, trace, Tracer } from "@internal/tracing"; +import { trail } from "agentcrumbs"; // @crumbs + +const crumb = trail("webapp:llm-dual-write"); // @crumbs import { createJsonErrorObject } from "@trigger.dev/core/v3/errors"; import { serializeTraceparent } from "@trigger.dev/core/v3/isomorphic"; import { @@ -94,6 +98,7 @@ export class ClickhouseEventRepository implements IEventRepository { private _clickhouse: ClickHouse; private _config: ClickhouseEventRepositoryConfig; private readonly _flushScheduler: DynamicFlushScheduler; + private readonly _llmUsageFlushScheduler: DynamicFlushScheduler; private _tracer: Tracer; private _version: "v1" | "v2"; @@ -118,6 +123,17 @@ export class ClickhouseEventRepository implements IEventRepository { return event.kind === "DEBUG_EVENT"; }, }); + + this._llmUsageFlushScheduler = new DynamicFlushScheduler({ + batchSize: 5000, + flushInterval: 2000, + callback: this.#flushLlmUsageBatch.bind(this), + minConcurrency: 1, + maxConcurrency: 2, + maxBatchSize: 10000, + memoryPressureThreshold: 10000, + loadSheddingEnabled: false, + }); } get version() { @@ -216,6 +232,58 @@ export class ClickhouseEventRepository implements IEventRepository { }); } + async #flushLlmUsageBatch(flushId: string, rows: LlmUsageV1Input[]) { + crumb("flushing llm usage batch", { flushId, rows: rows.length }); // @crumbs + + const [insertError] = await this._clickhouse.llmUsage.insert(rows, { + params: { + clickhouse_settings: this.#getClickhouseInsertSettings(), + }, + }); + + if (insertError) { + crumb("llm usage batch insert failed", { flushId, error: String(insertError) }); // @crumbs + throw insertError; + } + + crumb("llm usage batch inserted", { flushId, rows: rows.length }); // @crumbs + + logger.info("ClickhouseEventRepository.flushLlmUsageBatch Inserted LLM usage batch", { + rows: rows.length, + }); + } + + #createLlmUsageInput(event: CreateEventInput): LlmUsageV1Input { + const llmUsage = event._llmUsage!; + + return { + organization_id: event.organizationId, + project_id: event.projectId, + environment_id: event.environmentId, + run_id: event.runId, + task_identifier: event.taskSlug, + trace_id: event.traceId, + span_id: event.spanId, + gen_ai_system: llmUsage.genAiSystem, + request_model: llmUsage.requestModel, + response_model: llmUsage.responseModel, + matched_model_id: llmUsage.matchedModelId, + operation_name: llmUsage.operationName, + pricing_tier_id: llmUsage.pricingTierId, + pricing_tier_name: llmUsage.pricingTierName, + input_tokens: llmUsage.inputTokens, + output_tokens: llmUsage.outputTokens, + total_tokens: llmUsage.totalTokens, + usage_details: llmUsage.usageDetails, + input_cost: llmUsage.inputCost, + output_cost: llmUsage.outputCost, + total_cost: llmUsage.totalCost, + cost_details: llmUsage.costDetails, + start_time: this.#clampAndFormatStartTime(event.startTime.toString()), + duration: formatClickhouseUnsignedIntegerString(event.duration ?? 0), + }; + } + #getClickhouseInsertSettings() { if (this._config.insertStrategy === "insert") { return {}; @@ -236,6 +304,16 @@ export class ClickhouseEventRepository implements IEventRepository { async insertMany(events: CreateEventInput[]): Promise { this.addToBatch(events.flatMap((event) => this.createEventToTaskEventV1Input(event))); + + // Dual-write LLM usage records for spans with cost enrichment + const llmUsageRows = events + .filter((e) => e._llmUsage != null) + .map((e) => this.#createLlmUsageInput(e)); + + if (llmUsageRows.length > 0) { + crumb("queuing llm usage rows", { count: llmUsageRows.length, firstRunId: llmUsageRows[0]?.run_id }); // @crumbs + this._llmUsageFlushScheduler.addToBatch(llmUsageRows); + } } async insertManyImmediate(events: CreateEventInput[]): Promise { @@ -1525,7 +1603,13 @@ export class ClickhouseEventRepository implements IEventRepository { } if (parsedMetadata && "style" in parsedMetadata && parsedMetadata.style) { - span.data.style = parsedMetadata.style as TaskEventStyle; + const newStyle = parsedMetadata.style as TaskEventStyle; + // Merge styles: prefer the most complete value for each field + span.data.style = { + icon: newStyle.icon ?? span.data.style.icon, + variant: newStyle.variant ?? span.data.style.variant, + accessory: newStyle.accessory ?? span.data.style.accessory, + }; } if (record.kind === "SPAN") { diff --git a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts index 75664ad0525..b750786a651 100644 --- a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts +++ b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts @@ -21,6 +21,24 @@ export type { ExceptionEventProperties }; // Event Creation Types // ============================================================================ +export type LlmUsageData = { + genAiSystem: string; + requestModel: string; + responseModel: string; + matchedModelId: string; + operationName: string; + pricingTierId: string; + pricingTierName: string; + inputTokens: number; + outputTokens: number; + totalTokens: number; + usageDetails: Record; + inputCost: number; + outputCost: number; + totalCost: number; + costDetails: Record; +}; + export type CreateEventInput = Omit< Prisma.TaskEventCreateInput, | "id" @@ -57,6 +75,8 @@ export type CreateEventInput = Omit< metadata: Attributes | undefined; style: Attributes | undefined; machineId?: string; + /** Side-channel data for LLM cost tracking, populated by enrichCreatableEvents */ + _llmUsage?: LlmUsageData; }; export type CreatableEventKind = TaskEventKind; diff --git a/apps/webapp/app/v3/llmPricingRegistry.server.ts b/apps/webapp/app/v3/llmPricingRegistry.server.ts new file mode 100644 index 00000000000..4a2c67e247b --- /dev/null +++ b/apps/webapp/app/v3/llmPricingRegistry.server.ts @@ -0,0 +1,47 @@ +import { ModelPricingRegistry } from "@internal/llm-pricing"; +import { trail } from "agentcrumbs"; // @crumbs +import { $replica } from "~/db.server"; +import { env } from "~/env.server"; +import { singleton } from "~/utils/singleton"; +import { setLlmPricingRegistry } from "./utils/enrichCreatableEvents.server"; + +const crumb = trail("webapp:llm-registry"); // @crumbs + +export const llmPricingRegistry = singleton("llmPricingRegistry", () => { + if (!env.LLM_COST_TRACKING_ENABLED) { + crumb("llm cost tracking disabled via env"); // @crumbs + return null; + } + + crumb("initializing registry singleton"); // @crumbs + const registry = new ModelPricingRegistry($replica); + + // Wire up the registry so enrichCreatableEvents can use it + setLlmPricingRegistry(registry); + + registry + .loadFromDatabase() + .then(() => { + crumb("registry loaded successfully", { isLoaded: registry.isLoaded }); // @crumbs + }) + .catch((err) => { + crumb("registry load failed", { error: String(err) }); // @crumbs + console.error("Failed to load LLM pricing registry", err); + }); + + // Periodic reload + const reloadInterval = env.LLM_PRICING_RELOAD_INTERVAL_MS; + setInterval(() => { + registry + .reload() + .then(() => { + crumb("registry reloaded"); // @crumbs + }) + .catch((err) => { + crumb("registry reload failed", { error: String(err) }); // @crumbs + console.error("Failed to reload LLM pricing registry", err); + }); + }, reloadInterval); + + return registry; +}); diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index 837071b7de7..0079cb5f0be 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -37,6 +37,7 @@ import type { } from "./eventRepository/eventRepository.types"; import { startSpan } from "./tracing.server"; import { enrichCreatableEvents } from "./utils/enrichCreatableEvents.server"; +import "./llmPricingRegistry.server"; // Initialize LLM pricing registry on startup import { env } from "~/env.server"; import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings"; import { singleton } from "~/utils/singleton"; diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 53c3be60fa2..323310ebf84 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -599,7 +599,151 @@ export const metricsSchema: TableSchema = { /** * All available schemas for the query editor */ -export const querySchemas: TableSchema[] = [runsSchema, metricsSchema]; +/** + * Schema definition for the llm_usage table (trigger_dev.llm_usage_v1) + */ +export const llmUsageSchema: TableSchema = { + name: "llm_usage", + clickhouseName: "trigger_dev.llm_usage_v1", + description: "LLM token usage and cost data from GenAI spans", + timeConstraint: "start_time", + tenantColumns: { + organizationId: "organization_id", + projectId: "project_id", + environmentId: "environment_id", + }, + columns: { + environment: { + name: "environment", + clickhouseName: "environment_id", + ...column("String", { description: "The environment slug", example: "prod" }), + fieldMapping: "environment", + customRenderType: "environment", + }, + project: { + name: "project", + clickhouseName: "project_id", + ...column("String", { + description: "The project reference, they always start with `proj_`.", + example: "proj_howcnaxbfxdmwmxazktx", + }), + fieldMapping: "project", + customRenderType: "project", + }, + run_id: { + name: "run_id", + ...column("String", { + description: "The run ID", + customRenderType: "runId", + coreColumn: true, + }), + }, + task_identifier: { + name: "task_identifier", + ...column("LowCardinality(String)", { + description: "The task identifier", + example: "my-task", + coreColumn: true, + }), + }, + gen_ai_system: { + name: "gen_ai_system", + ...column("LowCardinality(String)", { + description: "AI provider (e.g. openai, anthropic)", + example: "openai", + coreColumn: true, + }), + }, + request_model: { + name: "request_model", + ...column("String", { + description: "The model name requested", + example: "gpt-4o", + }), + }, + response_model: { + name: "response_model", + ...column("String", { + description: "The model name returned by the provider", + example: "gpt-4o-2024-08-06", + coreColumn: true, + }), + }, + operation_name: { + name: "operation_name", + ...column("LowCardinality(String)", { + description: "Operation type (e.g. chat, completion)", + example: "chat", + }), + }, + input_tokens: { + name: "input_tokens", + ...column("UInt64", { + description: "Number of input tokens", + example: "702", + }), + }, + output_tokens: { + name: "output_tokens", + ...column("UInt64", { + description: "Number of output tokens", + example: "22", + }), + }, + total_tokens: { + name: "total_tokens", + ...column("UInt64", { + description: "Total token count", + example: "724", + }), + }, + input_cost: { + name: "input_cost", + ...column("Decimal64(12)", { + description: "Input cost in USD", + customRenderType: "costInDollars", + }), + }, + output_cost: { + name: "output_cost", + ...column("Decimal64(12)", { + description: "Output cost in USD", + customRenderType: "costInDollars", + }), + }, + total_cost: { + name: "total_cost", + ...column("Decimal64(12)", { + description: "Total cost in USD", + customRenderType: "costInDollars", + coreColumn: true, + }), + }, + pricing_tier_name: { + name: "pricing_tier_name", + ...column("LowCardinality(String)", { + description: "The matched pricing tier name", + example: "Standard", + }), + }, + start_time: { + name: "start_time", + ...column("DateTime64(9)", { + description: "When the LLM call started", + coreColumn: true, + }), + }, + duration: { + name: "duration", + ...column("UInt64", { + description: "Span duration in nanoseconds", + customRenderType: "durationNs", + }), + }, + }, +}; + +export const querySchemas: TableSchema[] = [runsSchema, metricsSchema, llmUsageSchema]; /** * Default query for the query editor diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index f718c13d2dd..9dfe091aa10 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -1,4 +1,31 @@ -import type { CreateEventInput } from "../eventRepository/eventRepository.types"; +import { trail } from "agentcrumbs"; // @crumbs +import type { CreateEventInput, LlmUsageData } from "../eventRepository/eventRepository.types"; + +const crumb = trail("webapp:llm-enrich"); // @crumbs + +// Registry interface — matches ModelPricingRegistry from @internal/llm-pricing +type CostRegistry = { + isLoaded: boolean; + calculateCost( + responseModel: string, + usageDetails: Record + ): { + matchedModelId: string; + matchedModelName: string; + pricingTierId: string; + pricingTierName: string; + inputCost: number; + outputCost: number; + totalCost: number; + costDetails: Record; + } | null; +}; + +let _registry: CostRegistry | undefined; + +export function setLlmPricingRegistry(registry: CostRegistry): void { + _registry = registry; +} export function enrichCreatableEvents(events: CreateEventInput[]) { return events.map((event) => { @@ -12,9 +39,151 @@ function enrichCreatableEvent(event: CreateEventInput): CreateEventInput { event.message = message; event.style = enrichStyle(event); + enrichLlmCost(event); + return event; } +function enrichLlmCost(event: CreateEventInput): void { + const props = event.properties; + if (!props) return; + + crumb("enrichLlmCost called", { kind: event.kind, isPartial: event.isPartial, spanId: event.spanId, message: event.message, props }); // @crumbs + + // Only enrich span-like events (INTERNAL, SERVER, CLIENT, CONSUMER, PRODUCER — not LOG, UNSPECIFIED) + const enrichableKinds = new Set(["INTERNAL", "SERVER", "CLIENT", "CONSUMER", "PRODUCER"]); + if (!enrichableKinds.has(event.kind as string)) return; + + // Skip partial spans (they don't have final token counts) + if (event.isPartial) return; + + // Only use gen_ai.* attributes for model resolution to avoid double-counting. + // The Vercel AI SDK emits both a parent span (ai.streamText with ai.usage.*) + // and a child span (ai.streamText.doStream with gen_ai.*). We only enrich the + // child span that has the canonical gen_ai.response.model attribute. + const responseModel = + typeof props["gen_ai.response.model"] === "string" + ? props["gen_ai.response.model"] + : typeof props["gen_ai.request.model"] === "string" + ? props["gen_ai.request.model"] + : null; + + if (!responseModel) { + // #region @crumbs + const genAiModel = props["gen_ai.response.model"]; + const aiModel = props["ai.model.id"]; + if (genAiModel || aiModel) { + crumb("responseModel null despite gen_ai attrs", { genAiModel, genAiModelType: typeof genAiModel, aiModel, spanId: event.spanId }); + } + // #endregion @crumbs + return; + } + + crumb("llm span detected", { responseModel, kind: event.kind, spanId: event.spanId }); // @crumbs + + // Extract usage details, normalizing attribute names + const usageDetails = extractUsageDetails(props); + + // Need at least some token usage + const hasTokens = Object.values(usageDetails).some((v) => v > 0); + if (!hasTokens) { + crumb("llm span skipped: no tokens", { responseModel, spanId: event.spanId }); // @crumbs + return; + } + + if (!_registry?.isLoaded) { + crumb("llm span skipped: registry not loaded", { responseModel }); // @crumbs + return; + } + + const cost = _registry.calculateCost(responseModel, usageDetails); + if (!cost) return; + + crumb("llm cost enriched", { responseModel, totalCost: cost.totalCost, matchedModel: cost.matchedModelName, spanId: event.spanId }); // @crumbs + + // Add trigger.llm.* attributes to the span + event.properties = { + ...props, + "trigger.llm.input_cost": cost.inputCost, + "trigger.llm.output_cost": cost.outputCost, + "trigger.llm.total_cost": cost.totalCost, + "trigger.llm.matched_model": cost.matchedModelName, + "trigger.llm.matched_model_id": cost.matchedModelId, + "trigger.llm.pricing_tier": cost.pricingTierName, + "trigger.llm.pricing_tier_id": cost.pricingTierId, + }; + + // Add style accessories for model, tokens, and cost + const inputTokens = usageDetails["input"] ?? 0; + const outputTokens = usageDetails["output"] ?? 0; + const totalTokens = inputTokens + outputTokens; + + event.style = { + ...event.style, + accessory: { + style: "pills", + items: [ + { text: responseModel, icon: "tabler-cube" }, + { text: formatTokenCount(totalTokens), icon: "tabler-hash" }, + { text: formatCost(cost.totalCost), icon: "tabler-currency-dollar" }, + ], + }, + }; + + // Set _llmUsage side-channel for dual-write to llm_usage_v1 + const llmUsage: LlmUsageData = { + genAiSystem: (props["gen_ai.system"] as string) ?? "unknown", + requestModel: (props["gen_ai.request.model"] as string) ?? responseModel, + responseModel, + matchedModelId: cost.matchedModelId, + operationName: (props["gen_ai.operation.name"] as string) ?? (props["operation.name"] as string) ?? "", + pricingTierId: cost.pricingTierId, + pricingTierName: cost.pricingTierName, + inputTokens: usageDetails["input"] ?? 0, + outputTokens: usageDetails["output"] ?? 0, + totalTokens: Object.values(usageDetails).reduce((sum, v) => sum + v, 0), + usageDetails, + inputCost: cost.inputCost, + outputCost: cost.outputCost, + totalCost: cost.totalCost, + costDetails: cost.costDetails, + }; + + event._llmUsage = llmUsage; +} + +function extractUsageDetails(props: Record): Record { + const details: Record = {}; + + // Only map gen_ai.usage.* attributes — NOT ai.usage.* from parent spans. + // This prevents double-counting when both parent (ai.streamText) and child + // (ai.streamText.doStream) spans carry token counts. + const mappings: Record = { + "gen_ai.usage.input_tokens": "input", + "gen_ai.usage.output_tokens": "output", + "gen_ai.usage.prompt_tokens": "input", + "gen_ai.usage.completion_tokens": "output", + "gen_ai.usage.total_tokens": "total", + "gen_ai.usage.cache_read_input_tokens": "input_cached_tokens", + "gen_ai.usage.input_tokens_cache_read": "input_cached_tokens", + "gen_ai.usage.cache_creation_input_tokens": "cache_creation_input_tokens", + "gen_ai.usage.input_tokens_cache_write": "cache_creation_input_tokens", + "gen_ai.usage.reasoning_tokens": "reasoning_tokens", + }; + + for (const [attrKey, usageKey] of Object.entries(mappings)) { + const value = props[attrKey]; + if (typeof value === "number" && value > 0) { + // Don't overwrite if already set (first mapping wins) + if (details[usageKey] === undefined) { + details[usageKey] = value; + } + } + } + + return details; +} + function enrichStyle(event: CreateEventInput) { const baseStyle = event.style ?? {}; const props = event.properties; @@ -49,6 +218,18 @@ function enrichStyle(event: CreateEventInput) { return baseStyle; } +function formatTokenCount(tokens: number): string { + if (tokens >= 1_000_000) return `${(tokens / 1_000_000).toFixed(1)}M`; + if (tokens >= 1_000) return `${(tokens / 1_000).toFixed(1)}k`; + return tokens.toString(); +} + +function formatCost(cost: number): string { + if (cost >= 1) return `$${cost.toFixed(2)}`; + if (cost >= 0.01) return `$${cost.toFixed(4)}`; + return `$${cost.toFixed(6)}`; +} + function repr(value: any): string { if (typeof value === "string") { return `'${value}'`; diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 139a0ce2d0d..fce7f2769c1 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -56,6 +56,7 @@ "@heroicons/react": "^2.0.12", "@jsonhero/schema-infer": "^0.1.5", "@internal/cache": "workspace:*", + "@internal/llm-pricing": "workspace:*", "@internal/redis": "workspace:*", "@internal/run-engine": "workspace:*", "@internal/schedule-engine": "workspace:*", diff --git a/apps/webapp/test/otlpExporter.test.ts b/apps/webapp/test/otlpExporter.test.ts index 98380f2a596..a1d1baa3b1c 100644 --- a/apps/webapp/test/otlpExporter.test.ts +++ b/apps/webapp/test/otlpExporter.test.ts @@ -1,5 +1,8 @@ -import { describe, it, expect } from "vitest"; -import { enrichCreatableEvents } from "../app/v3/utils/enrichCreatableEvents.server.js"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { + enrichCreatableEvents, + setLlmPricingRegistry, +} from "../app/v3/utils/enrichCreatableEvents.server.js"; import { RuntimeEnvironmentType, TaskEventKind, @@ -394,4 +397,237 @@ describe("OTLPExporter", () => { }); }); }); + + describe("LLM cost enrichment", () => { + const mockRegistry = { + isLoaded: true, + calculateCost: (responseModel: string, usageDetails: Record) => { + if (responseModel.startsWith("gpt-4o")) { + const inputCost = (usageDetails["input"] ?? 0) * 0.0000025; + const outputCost = (usageDetails["output"] ?? 0) * 0.00001; + return { + matchedModelId: "model-gpt4o", + matchedModelName: "gpt-4o", + pricingTierId: "tier-standard", + pricingTierName: "Standard", + inputCost, + outputCost, + totalCost: inputCost + outputCost, + costDetails: { input: inputCost, output: outputCost }, + }; + } + return null; + }, + }; + + beforeEach(() => { + setLlmPricingRegistry(mockRegistry); + }); + + afterEach(() => { + setLlmPricingRegistry(undefined as any); + }); + + function makeGenAiEvent(overrides: Record = {}) { + return { + message: "ai.streamText.doStream", + traceId: "test-trace", + spanId: "test-span", + parentId: "test-parent", + isPartial: false, + isError: false, + kind: TaskEventKind.INTERNAL, + level: TaskEventLevel.TRACE, + status: TaskEventStatus.UNSET, + startTime: BigInt(1), + duration: 5000000000, + style: {}, + serviceName: "test", + environmentId: "env-1", + environmentType: RuntimeEnvironmentType.DEVELOPMENT, + organizationId: "org-1", + projectId: "proj-1", + projectRef: "proj_test", + runId: "run_test", + runIsTest: false, + taskSlug: "my-task", + metadata: undefined, + properties: { + "gen_ai.system": "openai", + "gen_ai.request.model": "gpt-4o", + "gen_ai.response.model": "gpt-4o-2024-08-06", + "gen_ai.usage.input_tokens": 702, + "gen_ai.usage.output_tokens": 22, + "operation.name": "ai.streamText.doStream", + ...overrides, + }, + }; + } + + it("should enrich spans with cost attributes and accessories", () => { + const events = [makeGenAiEvent()]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + const event = $events[0]; + + // Cost attributes + expect(event.properties["trigger.llm.total_cost"]).toBeCloseTo(0.001975); + expect(event.properties["trigger.llm.input_cost"]).toBeCloseTo(0.001755); + expect(event.properties["trigger.llm.output_cost"]).toBeCloseTo(0.00022); + expect(event.properties["trigger.llm.matched_model"]).toBe("gpt-4o"); + expect(event.properties["trigger.llm.pricing_tier"]).toBe("Standard"); + + // Accessories (pills style) + expect(event.style.accessory).toBeDefined(); + expect(event.style.accessory.style).toBe("pills"); + expect(event.style.accessory.items).toHaveLength(3); + expect(event.style.accessory.items[0]).toEqual({ + text: "gpt-4o-2024-08-06", + icon: "tabler-cube", + }); + expect(event.style.accessory.items[1]).toEqual({ + text: "724", + icon: "tabler-hash", + }); + expect(event.style.accessory.items[2]).toEqual({ + text: "$0.001975", + icon: "tabler-currency-dollar", + }); + }); + + it("should set _llmUsage side-channel for dual-write", () => { + const events = [makeGenAiEvent()]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + const event = $events[0]; + + expect(event._llmUsage).toBeDefined(); + expect(event._llmUsage.genAiSystem).toBe("openai"); + expect(event._llmUsage.responseModel).toBe("gpt-4o-2024-08-06"); + expect(event._llmUsage.inputTokens).toBe(702); + expect(event._llmUsage.outputTokens).toBe(22); + expect(event._llmUsage.totalCost).toBeCloseTo(0.001975); + expect(event._llmUsage.operationName).toBe("ai.streamText.doStream"); + }); + + it("should skip partial spans", () => { + const events = [makeGenAiEvent()]; + events[0].isPartial = true; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + expect($events[0]._llmUsage).toBeUndefined(); + }); + + it("should skip spans without gen_ai.response.model or gen_ai.request.model", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.response.model": undefined, + "gen_ai.request.model": undefined, + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + }); + + it("should fall back to gen_ai.request.model when response.model is missing", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.response.model": undefined, + "gen_ai.request.model": "gpt-4o", + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.matched_model"]).toBe("gpt-4o"); + }); + + it("should skip spans with no token usage", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.usage.input_tokens": 0, + "gen_ai.usage.output_tokens": 0, + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + }); + + it("should skip spans with unknown models", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.response.model": "unknown-model-xyz", + "gen_ai.request.model": "unknown-model-xyz", + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + }); + + it("should not enrich non-span kinds like SPAN_EVENT or LOG", () => { + const events = [makeGenAiEvent()]; + events[0].kind = "SPAN_EVENT" as any; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + }); + + it("should enrich SERVER kind events", () => { + const events = [makeGenAiEvent()]; + events[0].kind = TaskEventKind.SERVER; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeCloseTo(0.001975); + }); + + it("should not enrich when registry is not loaded", () => { + setLlmPricingRegistry({ isLoaded: false, calculateCost: () => null }); + const events = [makeGenAiEvent()]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); + }); + + it("should format token counts with k/M suffixes in accessories", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.usage.input_tokens": 150000, + "gen_ai.usage.output_tokens": 2000, + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0].style.accessory.items[1].text).toBe("152.0k"); + }); + + it("should normalize alternate token attribute names", () => { + const events = [ + makeGenAiEvent({ + "gen_ai.usage.input_tokens": undefined, + "gen_ai.usage.output_tokens": undefined, + "gen_ai.usage.prompt_tokens": 500, + "gen_ai.usage.completion_tokens": 100, + }), + ]; + + // @ts-expect-error + const $events = enrichCreatableEvents(events); + expect($events[0]._llmUsage.inputTokens).toBe(500); + expect($events[0]._llmUsage.outputTokens).toBe(100); + }); + }); }); diff --git a/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql new file mode 100644 index 00000000000..65bd9c68ebd --- /dev/null +++ b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql @@ -0,0 +1,45 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS trigger_dev.llm_usage_v1 +( + organization_id LowCardinality(String), + project_id LowCardinality(String), + environment_id String CODEC(ZSTD(1)), + run_id String CODEC(ZSTD(1)), + task_identifier LowCardinality(String), + trace_id String CODEC(ZSTD(1)), + span_id String CODEC(ZSTD(1)), + + gen_ai_system LowCardinality(String), + request_model String CODEC(ZSTD(1)), + response_model String CODEC(ZSTD(1)), + matched_model_id String CODEC(ZSTD(1)), + operation_name LowCardinality(String), + pricing_tier_id String CODEC(ZSTD(1)), + pricing_tier_name LowCardinality(String), + + input_tokens UInt64 DEFAULT 0, + output_tokens UInt64 DEFAULT 0, + total_tokens UInt64 DEFAULT 0, + usage_details Map(LowCardinality(String), UInt64), + + input_cost Decimal64(12) DEFAULT 0, + output_cost Decimal64(12) DEFAULT 0, + total_cost Decimal64(12) DEFAULT 0, + cost_details Map(LowCardinality(String), Decimal64(12)), + + start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)), + duration UInt64 DEFAULT 0 CODEC(ZSTD(1)), + inserted_at DateTime64(3) DEFAULT now64(3), + + INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(inserted_at) +ORDER BY (organization_id, project_id, environment_id, toDate(inserted_at), run_id) +TTL toDateTime(inserted_at) + INTERVAL 365 DAY +SETTINGS ttl_only_drop_parts = 1; + +-- +goose Down +DROP TABLE IF EXISTS trigger_dev.llm_usage_v1; diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index b6fbd92177b..336abf3761f 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -27,6 +27,7 @@ import { getLogsSearchListQueryBuilder, } from "./taskEvents.js"; import { insertMetrics } from "./metrics.js"; +import { insertLlmUsage } from "./llmUsage.js"; import { getErrorGroups, getErrorInstances, @@ -44,6 +45,7 @@ import type { Agent as HttpsAgent } from "https"; export type * from "./taskRuns.js"; export type * from "./taskEvents.js"; export type * from "./metrics.js"; +export type * from "./llmUsage.js"; export type * from "./errors.js"; export type * from "./client/queryBuilder.js"; @@ -225,6 +227,12 @@ export class ClickHouse { }; } + get llmUsage() { + return { + insert: insertLlmUsage(this.writer), + }; + } + get taskEventsV2() { return { insert: insertTaskEventsV2(this.writer), diff --git a/internal-packages/clickhouse/src/llmUsage.ts b/internal-packages/clickhouse/src/llmUsage.ts new file mode 100644 index 00000000000..fd9deccb562 --- /dev/null +++ b/internal-packages/clickhouse/src/llmUsage.ts @@ -0,0 +1,42 @@ +import { z } from "zod"; +import { ClickhouseWriter } from "./client/types.js"; + +export const LlmUsageV1Input = z.object({ + organization_id: z.string(), + project_id: z.string(), + environment_id: z.string(), + run_id: z.string(), + task_identifier: z.string(), + trace_id: z.string(), + span_id: z.string(), + + gen_ai_system: z.string(), + request_model: z.string(), + response_model: z.string(), + matched_model_id: z.string(), + operation_name: z.string(), + pricing_tier_id: z.string(), + pricing_tier_name: z.string(), + + input_tokens: z.number(), + output_tokens: z.number(), + total_tokens: z.number(), + usage_details: z.record(z.string(), z.number()), + + input_cost: z.number(), + output_cost: z.number(), + total_cost: z.number(), + cost_details: z.record(z.string(), z.number()), + + start_time: z.string(), + duration: z.string(), +}); + +export type LlmUsageV1Input = z.input; + +export function insertLlmUsage(ch: ClickhouseWriter) { + return ch.insertUnsafe({ + name: "insertLlmUsage", + table: "trigger_dev.llm_usage_v1", + }); +} diff --git a/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql b/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql new file mode 100644 index 00000000000..44cc581616f --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql @@ -0,0 +1,63 @@ +-- CreateTable +CREATE TABLE "public"."llm_models" ( + "id" TEXT NOT NULL, + "project_id" TEXT, + "model_name" TEXT NOT NULL, + "match_pattern" TEXT NOT NULL, + "start_date" TIMESTAMP(3), + "source" TEXT NOT NULL DEFAULT 'default', + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "llm_models_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "public"."llm_pricing_tiers" ( + "id" TEXT NOT NULL, + "model_id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "is_default" BOOLEAN NOT NULL DEFAULT true, + "priority" INTEGER NOT NULL DEFAULT 0, + "conditions" JSONB NOT NULL DEFAULT '[]', + + CONSTRAINT "llm_pricing_tiers_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "public"."llm_prices" ( + "id" TEXT NOT NULL, + "model_id" TEXT NOT NULL, + "pricing_tier_id" TEXT NOT NULL, + "usage_type" TEXT NOT NULL, + "price" DECIMAL(20,12) NOT NULL, + + CONSTRAINT "llm_prices_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "llm_models_project_id_idx" ON "public"."llm_models"("project_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "llm_models_project_id_model_name_start_date_key" ON "public"."llm_models"("project_id", "model_name", "start_date"); + +-- CreateIndex +CREATE UNIQUE INDEX "llm_pricing_tiers_model_id_priority_key" ON "public"."llm_pricing_tiers"("model_id", "priority"); + +-- CreateIndex +CREATE UNIQUE INDEX "llm_pricing_tiers_model_id_name_key" ON "public"."llm_pricing_tiers"("model_id", "name"); + +-- CreateIndex +CREATE UNIQUE INDEX "llm_prices_model_id_usage_type_pricing_tier_id_key" ON "public"."llm_prices"("model_id", "usage_type", "pricing_tier_id"); + +-- AddForeignKey +ALTER TABLE "public"."llm_models" ADD CONSTRAINT "llm_models_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "public"."Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."llm_pricing_tiers" ADD CONSTRAINT "llm_pricing_tiers_model_id_fkey" FOREIGN KEY ("model_id") REFERENCES "public"."llm_models"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."llm_prices" ADD CONSTRAINT "llm_prices_model_id_fkey" FOREIGN KEY ("model_id") REFERENCES "public"."llm_models"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "public"."llm_prices" ADD CONSTRAINT "llm_prices_pricing_tier_id_fkey" FOREIGN KEY ("pricing_tier_id") REFERENCES "public"."llm_pricing_tiers"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index f6986be42c0..65878a8af7c 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -417,6 +417,7 @@ model Project { onboardingData Json? taskScheduleInstances TaskScheduleInstance[] metricsDashboards MetricsDashboard[] + llmModels LlmModel[] } enum ProjectVersion { @@ -2577,3 +2578,58 @@ model MetricsDashboard { /// Fast lookup for the list @@index([projectId, createdAt(sort: Desc)]) } + +// ==================================================== +// LLM Pricing Models +// ==================================================== + +/// A known LLM model or model pattern for cost tracking +model LlmModel { + id String @id @default(cuid()) + projectId String? @map("project_id") + project Project? @relation(fields: [projectId], references: [id], onDelete: Cascade) + modelName String @map("model_name") + matchPattern String @map("match_pattern") + startDate DateTime? @map("start_date") + source String @default("default") // "default", "admin", "project" + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @updatedAt @map("updated_at") + + pricingTiers LlmPricingTier[] + prices LlmPrice[] + + @@unique([projectId, modelName, startDate]) + @@index([projectId]) + @@map("llm_models") +} + +/// A pricing tier for a model (supports volume-based or conditional pricing) +model LlmPricingTier { + id String @id @default(cuid()) + modelId String @map("model_id") + model LlmModel @relation(fields: [modelId], references: [id], onDelete: Cascade) + name String + isDefault Boolean @default(true) @map("is_default") + priority Int @default(0) + conditions Json @default("[]") @db.JsonB + + prices LlmPrice[] + + @@unique([modelId, priority]) + @@unique([modelId, name]) + @@map("llm_pricing_tiers") +} + +/// A price point for a usage type within a pricing tier +model LlmPrice { + id String @id @default(cuid()) + modelId String @map("model_id") + model LlmModel @relation(fields: [modelId], references: [id], onDelete: Cascade) + pricingTierId String @map("pricing_tier_id") + pricingTier LlmPricingTier @relation(fields: [pricingTierId], references: [id], onDelete: Cascade) + usageType String @map("usage_type") + price Decimal @db.Decimal(20, 12) + + @@unique([modelId, usageType, pricingTierId]) + @@map("llm_prices") +} diff --git a/internal-packages/llm-pricing/package.json b/internal-packages/llm-pricing/package.json new file mode 100644 index 00000000000..423dae77eb4 --- /dev/null +++ b/internal-packages/llm-pricing/package.json @@ -0,0 +1,17 @@ +{ + "name": "@internal/llm-pricing", + "private": true, + "version": "0.0.1", + "main": "./src/index.ts", + "types": "./src/index.ts", + "type": "module", + "dependencies": { + "@trigger.dev/database": "workspace:*" + }, + "scripts": { + "typecheck": "tsc --noEmit", + "generate": "./scripts/sync-model-prices.sh", + "sync-prices": "./scripts/sync-model-prices.sh", + "sync-prices:check": "./scripts/sync-model-prices.sh --check" + } +} diff --git a/internal-packages/llm-pricing/scripts/sync-model-prices.sh b/internal-packages/llm-pricing/scripts/sync-model-prices.sh new file mode 100755 index 00000000000..9bb6af11cbb --- /dev/null +++ b/internal-packages/llm-pricing/scripts/sync-model-prices.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Sync default model prices from Langfuse's repository and generate the TS module. +# Usage: ./scripts/sync-model-prices.sh [--check] +# --check: Exit 1 if prices are outdated (for CI) + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PACKAGE_DIR="$(dirname "$SCRIPT_DIR")" +JSON_TARGET="$PACKAGE_DIR/src/default-model-prices.json" +TS_TARGET="$PACKAGE_DIR/src/defaultPrices.ts" +SOURCE_URL="https://raw.githubusercontent.com/langfuse/langfuse/main/worker/src/constants/default-model-prices.json" + +CHECK_MODE=false +if [[ "${1:-}" == "--check" ]]; then + CHECK_MODE=true +fi + +echo "Fetching latest model prices from Langfuse..." +TMPFILE=$(mktemp) +trap 'rm -f "$TMPFILE"' EXIT + +if ! curl -fsSL "$SOURCE_URL" -o "$TMPFILE"; then + echo "ERROR: Failed to fetch from $SOURCE_URL" + exit 1 +fi + +# Validate it's valid JSON with at least some models +MODEL_COUNT=$(node -e "console.log(JSON.parse(require('fs').readFileSync('$TMPFILE','utf-8')).length)" 2>/dev/null || echo "0") +if [[ "$MODEL_COUNT" -lt 10 ]]; then + echo "ERROR: Downloaded file has only $MODEL_COUNT models (expected 100+). Aborting." + exit 1 +fi + +if $CHECK_MODE; then + if diff -q "$JSON_TARGET" "$TMPFILE" > /dev/null 2>&1; then + echo "Model prices are up to date ($MODEL_COUNT models)" + exit 0 + else + echo "Model prices are OUTDATED. Run 'pnpm run sync-prices' in @internal/llm-pricing to update." + exit 1 + fi +fi + +cp "$TMPFILE" "$JSON_TARGET" +echo "Updated default-model-prices.json ($MODEL_COUNT models)" + +# Generate the TypeScript module from the JSON +echo "Generating defaultPrices.ts..." +node -e " +const data = JSON.parse(require('fs').readFileSync('$JSON_TARGET', 'utf-8')); +const stripped = data.map(e => ({ + modelName: e.modelName, + matchPattern: e.matchPattern, + startDate: e.createdAt, + pricingTiers: e.pricingTiers.map(t => ({ + name: t.name, + isDefault: t.isDefault, + priority: t.priority, + conditions: t.conditions.map(c => ({ + usageDetailPattern: c.usageDetailPattern, + operator: c.operator, + value: c.value, + })), + prices: t.prices, + })), +})); + +let out = 'import type { DefaultModelDefinition } from \"./types.js\";\n\n'; +out += '// Auto-generated from Langfuse default-model-prices.json — do not edit manually.\n'; +out += '// Run \`pnpm run sync-prices\` to update from upstream.\n'; +out += '// Source: https://github.com/langfuse/langfuse\n\n'; +out += 'export const defaultModelPrices: DefaultModelDefinition[] = '; +out += JSON.stringify(stripped, null, 2) + ';\n'; +require('fs').writeFileSync('$TS_TARGET', out); +console.log('Generated defaultPrices.ts with ' + stripped.length + ' models'); +" diff --git a/internal-packages/llm-pricing/src/default-model-prices.json b/internal-packages/llm-pricing/src/default-model-prices.json new file mode 100644 index 00000000000..486c6c512b5 --- /dev/null +++ b/internal-packages/llm-pricing/src/default-model-prices.json @@ -0,0 +1,3838 @@ +[ + { + "id": "b9854a5c92dc496b997d99d20", + "modelName": "gpt-4o", + "matchPattern": "(?i)^(openai\/)?(gpt-4o)$", + "createdAt": "2024-05-13T23:15:07.670Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "b9854a5c92dc496b997d99d20_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "id": "b9854a5c92dc496b997d99d21", + "modelName": "gpt-4o-2024-05-13", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-2024-05-13)$", + "createdAt": "2024-05-13T23:15:07.670Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o-2024-05-13", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "b9854a5c92dc496b997d99d21_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "output": 0.000015 + } + } + ] + }, + { + "id": "clrkvq6iq000008ju6c16gynt", + "modelName": "gpt-4-1106-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4-1106-preview)$", + "createdAt": "2024-04-23T10:37:17.092Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-1106-preview", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkvq6iq000008ju6c16gynt_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "clrkvx5gp000108juaogs54ea", + "modelName": "gpt-4-turbo-vision", + "matchPattern": "(?i)^(openai\/)?(gpt-4(-\\d{4})?-vision-preview)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-vision-preview", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkvx5gp000108juaogs54ea_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "clrkvyzgw000308jue4hse4j9", + "modelName": "gpt-4-32k", + "matchPattern": "(?i)^(openai\/)?(gpt-4-32k)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-32k", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkvyzgw000308jue4hse4j9_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "id": "clrkwk4cb000108l5hwwh3zdi", + "modelName": "gpt-4-32k-0613", + "matchPattern": "(?i)^(openai\/)?(gpt-4-32k-0613)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-32k-0613", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkwk4cb000108l5hwwh3zdi_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "id": "clrkwk4cb000208l59yvb9yq8", + "modelName": "gpt-3.5-turbo-1106", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-1106)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-1106", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkwk4cb000208l59yvb9yq8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000001, + "output": 0.000002 + } + } + ] + }, + { + "id": "clrkwk4cc000808l51xmk4uic", + "modelName": "gpt-3.5-turbo-0613", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-0613)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-0613", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkwk4cc000808l51xmk4uic_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000015, + "output": 0.000002 + } + } + ] + }, + { + "id": "clrkwk4cc000908l537kl0rx3", + "modelName": "gpt-4-0613", + "matchPattern": "(?i)^(openai\/)?(gpt-4-0613)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-0613", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkwk4cc000908l537kl0rx3_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "id": "clrkwk4cc000a08l562uc3s9g", + "modelName": "gpt-3.5-turbo-instruct", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-instruct)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrkwk4cc000a08l562uc3s9g_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000015, + "output": 0.000002 + } + } + ] + }, + { + "id": "clrntjt89000108jwcou1af71", + "modelName": "text-ada-001", + "matchPattern": "(?i)^(text-ada-001)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-ada-001" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000108jwcou1af71_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.000004 + } + } + ] + }, + { + "id": "clrntjt89000208jwawjr894q", + "modelName": "text-babbage-001", + "matchPattern": "(?i)^(text-babbage-001)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-babbage-001" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000208jwawjr894q_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 5e-7 + } + } + ] + }, + { + "id": "clrntjt89000308jw0jtfa4rs", + "modelName": "text-curie-001", + "matchPattern": "(?i)^(text-curie-001)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-curie-001" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000308jw0jtfa4rs_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "id": "clrntjt89000408jwc2c93h6i", + "modelName": "text-davinci-001", + "matchPattern": "(?i)^(text-davinci-001)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-davinci-001" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000408jwc2c93h6i_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "id": "clrntjt89000508jw192m64qi", + "modelName": "text-davinci-002", + "matchPattern": "(?i)^(text-davinci-002)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-davinci-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000508jw192m64qi_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "id": "clrntjt89000608jw4m3x5s55", + "modelName": "text-davinci-003", + "matchPattern": "(?i)^(text-davinci-003)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-davinci-003" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000608jw4m3x5s55_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "id": "clrntjt89000908jwhvkz5crg", + "modelName": "text-embedding-ada-002-v2", + "matchPattern": "(?i)^(text-embedding-ada-002-v2)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-embedding-ada-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000908jwhvkz5crg_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "id": "clrntjt89000908jwhvkz5crm", + "modelName": "text-embedding-ada-002", + "matchPattern": "(?i)^(text-embedding-ada-002)$", + "createdAt": "2024-01-24T18:18:50.861Z", + "updatedAt": "2024-01-24T18:18:50.861Z", + "tokenizerConfig": { + "tokenizerModel": "text-embedding-ada-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000908jwhvkz5crm_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "id": "clrntjt89000a08jw0gcdbd5a", + "modelName": "gpt-3.5-turbo-16k-0613", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-16k-0613)$", + "createdAt": "2024-02-03T17:29:57.350Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-16k-0613", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntjt89000a08jw0gcdbd5a_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "output": 0.000004 + } + } + ] + }, + { + "id": "clrntkjgy000a08jx4e062mr0", + "modelName": "gpt-3.5-turbo-0301", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-0301)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": -1, + "tokenizerModel": "gpt-3.5-turbo-0301", + "tokensPerMessage": 4 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntkjgy000a08jx4e062mr0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "output": 0.000002 + } + } + ] + }, + { + "id": "clrntkjgy000d08jx0p4y9h4l", + "modelName": "gpt-4-32k-0314", + "matchPattern": "(?i)^(openai\/)?(gpt-4-32k-0314)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-32k-0314", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntkjgy000d08jx0p4y9h4l_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "id": "clrntkjgy000e08jx4x6uawoo", + "modelName": "gpt-4-0314", + "matchPattern": "(?i)^(openai\/)?(gpt-4-0314)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-0314", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntkjgy000e08jx4x6uawoo_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "id": "clrntkjgy000f08jx79v9g1xj", + "modelName": "gpt-4", + "matchPattern": "(?i)^(openai\/)?(gpt-4)$", + "createdAt": "2024-01-24T10:19:21.693Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrntkjgy000f08jx79v9g1xj_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "id": "clrnwb41q000308jsfrac9uh6", + "modelName": "claude-instant-1.2", + "matchPattern": "(?i)^(anthropic\/)?(claude-instant-1.2)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwb41q000308jsfrac9uh6_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000163, + "output": 0.00000551 + } + } + ] + }, + { + "id": "clrnwb836000408jsallr6u11", + "modelName": "claude-2.0", + "matchPattern": "(?i)^(anthropic\/)?(claude-2.0)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwb836000408jsallr6u11_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "id": "clrnwbd1m000508js4hxu6o7n", + "modelName": "claude-2.1", + "matchPattern": "(?i)^(anthropic\/)?(claude-2.1)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwbd1m000508js4hxu6o7n_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "id": "clrnwbg2b000608jse2pp4q2d", + "modelName": "claude-1.3", + "matchPattern": "(?i)^(anthropic\/)?(claude-1.3)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwbg2b000608jse2pp4q2d_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "id": "clrnwbi9d000708jseiy44k26", + "modelName": "claude-1.2", + "matchPattern": "(?i)^(anthropic\/)?(claude-1.2)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwbi9d000708jseiy44k26_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "id": "clrnwblo0000808jsc1385hdp", + "modelName": "claude-1.1", + "matchPattern": "(?i)^(anthropic\/)?(claude-1.1)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwblo0000808jsc1385hdp_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "id": "clrnwbota000908jsgg9mb1ml", + "modelName": "claude-instant-1", + "matchPattern": "(?i)^(anthropic\/)?(claude-instant-1)$", + "createdAt": "2024-01-30T15:44:13.447Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clrnwbota000908jsgg9mb1ml_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000163, + "output": 0.00000551 + } + } + ] + }, + { + "id": "clrs2dnql000108l46vo0gp2t", + "modelName": "babbage-002", + "matchPattern": "(?i)^(babbage-002)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2024-01-26T17:35:21.129Z", + "tokenizerConfig": { + "tokenizerModel": "babbage-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrs2dnql000108l46vo0gp2t_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "output": 0.0000016 + } + } + ] + }, + { + "id": "clrs2ds35000208l4g4b0hi3u", + "modelName": "davinci-002", + "matchPattern": "(?i)^(davinci-002)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2024-01-26T17:35:21.129Z", + "tokenizerConfig": { + "tokenizerModel": "davinci-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clrs2ds35000208l4g4b0hi3u_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000006, + "output": 0.000012 + } + } + ] + }, + { + "id": "clruwn3pc00010al7bl611c8o", + "modelName": "text-embedding-3-small", + "matchPattern": "(?i)^(text-embedding-3-small)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2024-01-26T17:35:21.129Z", + "tokenizerConfig": { + "tokenizerModel": "text-embedding-ada-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clruwn3pc00010al7bl611c8o_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 2e-8 + } + } + ] + }, + { + "id": "clruwn76700020al7gp8e4g4l", + "modelName": "text-embedding-3-large", + "matchPattern": "(?i)^(text-embedding-3-large)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2024-01-26T17:35:21.129Z", + "tokenizerConfig": { + "tokenizerModel": "text-embedding-ada-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clruwn76700020al7gp8e4g4l_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1.3e-7 + } + } + ] + }, + { + "id": "clruwnahl00030al7ab9rark7", + "modelName": "gpt-3.5-turbo-0125", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-0125)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clruwnahl00030al7ab9rark7_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "id": "clruwnahl00040al78f1lb0at", + "modelName": "gpt-3.5-turbo", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo)$", + "createdAt": "2024-02-13T12:00:37.424Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clruwnahl00040al78f1lb0at_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "id": "clruwnahl00050al796ck3p44", + "modelName": "gpt-4-0125-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4-0125-preview)$", + "createdAt": "2024-01-26T17:35:21.129Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clruwnahl00050al796ck3p44_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "cls08r8sq000308jq14ae96f0", + "modelName": "ft:gpt-3.5-turbo-1106", + "matchPattern": "(?i)^(ft:)(gpt-3.5-turbo-1106:)(.+)(:)(.*)(:)(.+)$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-1106", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cls08r8sq000308jq14ae96f0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "output": 0.000006 + } + } + ] + }, + { + "id": "cls08rp99000408jqepxoakjv", + "modelName": "ft:gpt-3.5-turbo-0613", + "matchPattern": "(?i)^(ft:)(gpt-3.5-turbo-0613:)(.+)(:)(.*)(:)(.+)$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-0613", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cls08rp99000408jqepxoakjv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000012, + "output": 0.000016 + } + } + ] + }, + { + "id": "cls08rv9g000508jq5p4z4nlr", + "modelName": "ft:davinci-002", + "matchPattern": "(?i)^(ft:)(davinci-002:)(.+)(:)(.*)(:)(.+)$$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "tokenizerConfig": { + "tokenizerModel": "davinci-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cls08rv9g000508jq5p4z4nlr_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000012, + "output": 0.000012 + } + } + ] + }, + { + "id": "cls08s2bw000608jq57wj4un2", + "modelName": "ft:babbage-002", + "matchPattern": "(?i)^(ft:)(babbage-002:)(.+)(:)(.*)(:)(.+)$$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "tokenizerConfig": { + "tokenizerModel": "babbage-002" + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cls08s2bw000608jq57wj4un2_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000016, + "output": 0.0000016 + } + } + ] + }, + { + "id": "cls0iv12d000108l251gf3038", + "modelName": "chat-bison", + "matchPattern": "(?i)^(chat-bison)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0iv12d000108l251gf3038_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls0j33v1000008joagkc4lql", + "modelName": "codechat-bison-32k", + "matchPattern": "(?i)^(codechat-bison-32k)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0j33v1000008joagkc4lql_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls0jmc9v000008l8ee6r3gsd", + "modelName": "codechat-bison", + "matchPattern": "(?i)^(codechat-bison)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0jmc9v000008l8ee6r3gsd_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls0jmjt3000108l83ix86w0d", + "modelName": "text-bison-32k", + "matchPattern": "(?i)^(text-bison-32k)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0jmjt3000108l83ix86w0d_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls0jni4t000008jk3kyy803r", + "modelName": "chat-bison-32k", + "matchPattern": "(?i)^(chat-bison-32k)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0jni4t000008jk3kyy803r_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls0jungb000208jk12gm4gk1", + "modelName": "text-unicorn", + "matchPattern": "(?i)^(text-unicorn)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0jungb000208jk12gm4gk1_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "output": 0.0000075 + } + } + ] + }, + { + "id": "cls0juygp000308jk2a6x9my2", + "modelName": "text-bison", + "matchPattern": "(?i)^(text-bison)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls0juygp000308jk2a6x9my2_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls1nyj5q000208l33ne901d8", + "modelName": "textembedding-gecko", + "matchPattern": "(?i)^(textembedding-gecko)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls1nyj5q000208l33ne901d8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "id": "cls1nyyjp000308l31gxy1bih", + "modelName": "textembedding-gecko-multilingual", + "matchPattern": "(?i)^(textembedding-gecko-multilingual)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls1nyyjp000308l31gxy1bih_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "id": "cls1nzjt3000508l3dnwad3g0", + "modelName": "code-gecko", + "matchPattern": "(?i)^(code-gecko)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls1nzjt3000508l3dnwad3g0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls1nzwx4000608l38va7e4tv", + "modelName": "code-bison", + "matchPattern": "(?i)^(code-bison)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls1nzwx4000608l38va7e4tv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cls1o053j000708l39f8g4bgs", + "modelName": "code-bison-32k", + "matchPattern": "(?i)^(code-bison-32k)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-01-31T13:25:02.141Z", + "updatedAt": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "id": "cls1o053j000708l39f8g4bgs_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "clsk9lntu000008jwfc51bbqv", + "modelName": "gpt-3.5-turbo-16k", + "matchPattern": "(?i)^(openai\/)?(gpt-)(35|3.5)(-turbo-16k)$", + "createdAt": "2024-02-13T12:00:37.424Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-3.5-turbo-16k", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clsk9lntu000008jwfc51bbqv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "id": "clsnq07bn000008l4e46v1ll8", + "modelName": "gpt-4-turbo-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4-turbo-preview)$", + "createdAt": "2024-02-15T21:21:50.947Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clsnq07bn000008l4e46v1ll8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "cltgy0iuw000008le3vod1hhy", + "modelName": "claude-3-opus-20240229", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-opus-20240229|anthropic\\.claude-3-opus-20240229-v1:0|claude-3-opus@20240229)$", + "createdAt": "2024-03-07T17:55:38.139Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cltgy0iuw000008le3vod1hhy_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.000075 + } + } + ] + }, + { + "id": "cltgy0pp6000108le56se7bl3", + "modelName": "claude-3-sonnet-20240229", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-sonnet-20240229|anthropic\\.claude-3-sonnet-20240229-v1:0|claude-3-sonnet@20240229)$", + "createdAt": "2024-03-07T17:55:38.139Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cltgy0pp6000108le56se7bl3_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cltr0w45b000008k1407o9qv1", + "modelName": "claude-3-haiku-20240307", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-haiku-20240307|anthropic\\.claude-3-haiku-20240307-v1:0|claude-3-haiku@20240307)$", + "createdAt": "2024-03-14T09:41:18.736Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cltr0w45b000008k1407o9qv1_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 0.00000125 + } + } + ] + }, + { + "id": "cluv2sjeo000008ih0fv23hi0", + "modelName": "gemini-1.0-pro-latest", + "matchPattern": "(?i)^(google\/)?(gemini-1.0-pro-latest)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-04-11T10:27:46.517Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cluv2sjeo000008ih0fv23hi0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "id": "cluv2subq000108ih2mlrga6a", + "modelName": "gemini-1.0-pro", + "matchPattern": "(?i)^(google\/)?(gemini-1.0-pro)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-04-11T10:27:46.517Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cluv2subq000108ih2mlrga6a_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "id": "cluv2sx04000208ihbek75lsz", + "modelName": "gemini-1.0-pro-001", + "matchPattern": "(?i)^(google\/)?(gemini-1.0-pro-001)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-04-11T10:27:46.517Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cluv2sx04000208ihbek75lsz_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "id": "cluv2szw0000308ihch3n79x7", + "modelName": "gemini-pro", + "matchPattern": "(?i)^(google\/)?(gemini-pro)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-04-11T10:27:46.517Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cluv2szw0000308ihch3n79x7_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "id": "cluv2t2x0000408ihfytl45l1", + "modelName": "gemini-1.5-pro-latest", + "matchPattern": "(?i)^(google\/)?(gemini-1.5-pro-latest)(@[a-zA-Z0-9]+)?$", + "createdAt": "2024-04-11T10:27:46.517Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cluv2t2x0000408ihfytl45l1_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "output": 0.0000075 + } + } + ] + }, + { + "id": "cluv2t5k3000508ih5kve9zag", + "modelName": "gpt-4-turbo-2024-04-09", + "matchPattern": "(?i)^(openai\/)?(gpt-4-turbo-2024-04-09)$", + "createdAt": "2024-04-23T10:37:17.092Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-turbo-2024-04-09", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cluv2t5k3000508ih5kve9zag_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "cluvpl4ls000008l6h2gx3i07", + "modelName": "gpt-4-turbo", + "matchPattern": "(?i)^(openai\/)?(gpt-4-turbo)$", + "createdAt": "2024-04-11T21:13:44.989Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-1106-preview", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cluvpl4ls000008l6h2gx3i07_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "clv2o2x0p000008jsf9afceau", + "modelName": " gpt-4-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4-preview)$", + "createdAt": "2024-04-23T10:37:17.092Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4-turbo-preview", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clv2o2x0p000008jsf9afceau_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "id": "clxt0n0m60000pumz1j5b7zsf", + "modelName": "claude-3-5-sonnet-20240620", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-5-sonnet-20240620|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20240620-v1:0|claude-3-5-sonnet@20240620)$", + "createdAt": "2024-06-25T11:47:24.475Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "clxt0n0m60000pumz1j5b7zsf_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "clyrjp56f0000t0mzapoocd7u", + "modelName": "gpt-4o-mini", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-mini)$", + "createdAt": "2024-07-18T17:56:09.591Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clyrjp56f0000t0mzapoocd7u_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.5e-7, + "output": 6e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8 + } + } + ] + }, + { + "id": "clyrjpbe20000t0mzcbwc42rg", + "modelName": "gpt-4o-mini-2024-07-18", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-mini-2024-07-18)$", + "createdAt": "2024-07-18T17:56:09.591Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clyrjpbe20000t0mzcbwc42rg_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.5e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8, + "output": 6e-7 + } + } + ] + }, + { + "id": "clzjr85f70000ymmzg7hqffra", + "modelName": "gpt-4o-2024-08-06", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-2024-08-06)$", + "createdAt": "2024-08-07T11:54:31.298Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "clzjr85f70000ymmzg7hqffra_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "id": "cm10ivcdp0000gix7lelmbw80", + "modelName": "o1-preview", + "matchPattern": "(?i)^(openai\/)?(o1-preview)$", + "createdAt": "2024-09-13T10:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm10ivcdp0000gix7lelmbw80_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "id": "cm10ivo130000n8x7qopcjjcg", + "modelName": "o1-preview-2024-09-12", + "matchPattern": "(?i)^(openai\/)?(o1-preview-2024-09-12)$", + "createdAt": "2024-09-13T10:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm10ivo130000n8x7qopcjjcg_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "id": "cm10ivwo40000r1x7gg3syjq0", + "modelName": "o1-mini", + "matchPattern": "(?i)^(openai\/)?(o1-mini)$", + "createdAt": "2024-09-13T10:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm10ivwo40000r1x7gg3syjq0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm10iw6p20000wgx7it1hlb22", + "modelName": "o1-mini-2024-09-12", + "matchPattern": "(?i)^(openai\/)?(o1-mini-2024-09-12)$", + "createdAt": "2024-09-13T10:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm10iw6p20000wgx7it1hlb22_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm2krz1uf000208jjg5653iud", + "modelName": "claude-3.5-sonnet-20241022", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-5-sonnet-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20241022-v2:0|claude-3-5-sonnet-V2@20241022)$", + "createdAt": "2024-10-22T18:48:01.676Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm2krz1uf000208jjg5653iud_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cm2ks2vzn000308jjh4ze1w7q", + "modelName": "claude-3.5-sonnet-latest", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-5-sonnet-latest)$", + "createdAt": "2024-10-22T18:48:01.676Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm2ks2vzn000308jjh4ze1w7q_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cm34aq60d000207ml0j1h31ar", + "modelName": "claude-3-5-haiku-20241022", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-5-haiku-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-haiku-20241022-v1:0|claude-3-5-haiku-V1@20241022)$", + "createdAt": "2024-11-05T10:30:50.566Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm34aq60d000207ml0j1h31ar_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 8e-7, + "input_tokens": 8e-7, + "output": 0.000004, + "output_tokens": 0.000004, + "cache_creation_input_tokens": 0.000001, + "input_cache_creation": 0.000001, + "input_cache_creation_5m": 0.000001, + "input_cache_creation_1h": 0.0000016, + "cache_read_input_tokens": 8e-8, + "input_cache_read": 8e-8 + } + } + ] + }, + { + "id": "cm34aqb9h000307ml6nypd618", + "modelName": "claude-3.5-haiku-latest", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-5-haiku-latest)$", + "createdAt": "2024-11-05T10:30:50.566Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm34aqb9h000307ml6nypd618_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 8e-7, + "input_tokens": 8e-7, + "output": 0.000004, + "output_tokens": 0.000004, + "cache_creation_input_tokens": 0.000001, + "input_cache_creation": 0.000001, + "input_cache_creation_5m": 0.000001, + "input_cache_creation_1h": 0.0000016, + "cache_read_input_tokens": 8e-8, + "input_cache_read": 8e-8 + } + } + ] + }, + { + "id": "cm3x0p8ev000008kyd96800c8", + "modelName": "chatgpt-4o-latest", + "matchPattern": "(?i)^(chatgpt-4o-latest)$", + "createdAt": "2024-11-25T12:47:17.504Z", + "updatedAt": "2024-11-25T12:47:17.504Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm3x0p8ev000008kyd96800c8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "output": 0.000015 + } + } + ] + }, + { + "id": "cm48akqgo000008ldbia24qg0", + "modelName": "gpt-4o-2024-11-20", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-2024-11-20)$", + "createdAt": "2024-12-03T10:06:12.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4o", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm48akqgo000008ldbia24qg0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "id": "cm48b2ksh000008l0hn3u0hl3", + "modelName": "gpt-4o-audio-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-audio-preview)$", + "createdAt": "2024-12-03T10:19:56.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48b2ksh000008l0hn3u0hl3_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.0000025, + "output_text_tokens": 0.00001, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "id": "cm48bbm0k000008l69nsdakwf", + "modelName": "gpt-4o-audio-preview-2024-10-01", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-audio-preview-2024-10-01)$", + "createdAt": "2024-12-03T10:19:56.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48bbm0k000008l69nsdakwf_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.0000025, + "output_text_tokens": 0.00001, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "id": "cm48c2qh4000008mhgy4mg2qc", + "modelName": "gpt-4o-realtime-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-realtime-preview)$", + "createdAt": "2024-12-03T10:19:56.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48c2qh4000008mhgy4mg2qc_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.000005, + "input_cached_text_tokens": 0.0000025, + "output_text_tokens": 0.00002, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "input_cached_audio_tokens": 0.00002, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "id": "cm48cjxtc000008jrcsso3avv", + "modelName": "gpt-4o-realtime-preview-2024-10-01", + "matchPattern": "(?i)^(openai\/)?(gpt-4o-realtime-preview-2024-10-01)$", + "createdAt": "2024-12-03T10:19:56.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48cjxtc000008jrcsso3avv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.000005, + "input_cached_text_tokens": 0.0000025, + "output_text_tokens": 0.00002, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "input_cached_audio_tokens": 0.00002, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "id": "cm48cjxtc000108jrcsso3avv", + "modelName": "o1", + "matchPattern": "(?i)^(openai\/)?(o1)$", + "createdAt": "2025-01-17T00:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48cjxtc000108jrcsso3avv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "id": "cm48cjxtc000208jrcsso3avv", + "modelName": "o1-2024-12-17", + "matchPattern": "(?i)^(openai\/)?(o1-2024-12-17)$", + "createdAt": "2025-01-17T00:01:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm48cjxtc000208jrcsso3avv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "id": "cm6l8j7vs0000tymz9vk7ew8t", + "modelName": "o3-mini", + "matchPattern": "(?i)^(openai\/)?(o3-mini)$", + "createdAt": "2025-01-31T20:41:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm6l8j7vs0000tymz9vk7ew8t_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm6l8jan90000tymz52sh0ql8", + "modelName": "o3-mini-2025-01-31", + "matchPattern": "(?i)^(openai\/)?(o3-mini-2025-01-31)$", + "createdAt": "2025-01-31T20:41:35.373Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm6l8jan90000tymz52sh0ql8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm6l8jdef0000tymz52sh0ql0", + "modelName": "gemini-2.0-flash-001", + "matchPattern": "(?i)^(google\/)?(gemini-2.0-flash-001)(@[a-zA-Z0-9]+)?$", + "createdAt": "2025-02-06T11:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm6l8jdef0000tymz52sh0ql0_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "output": 4e-7 + } + } + ] + }, + { + "id": "cm6l8jfgh0000tymz52sh0ql1", + "modelName": "gemini-2.0-flash-lite-preview-02-05", + "matchPattern": "(?i)^(google\/)?(gemini-2.0-flash-lite-preview-02-05)(@[a-zA-Z0-9]+)?$", + "createdAt": "2025-02-06T11:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm6l8jfgh0000tymz52sh0ql1_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-8, + "output": 3e-7 + } + } + ] + }, + { + "id": "cm7ka7561000108js3t9tb3at", + "modelName": "claude-3.7-sonnet-20250219", + "matchPattern": "(?i)^(anthropic\/)?(claude-3.7-sonnet-20250219|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3.7-sonnet-20250219-v1:0|claude-3-7-sonnet-V1@20250219)$", + "createdAt": "2025-02-25T09:35:39.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm7ka7561000108js3t9tb3at_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cm7ka7zob000208jsfs9h5ajj", + "modelName": "claude-3.7-sonnet-latest", + "matchPattern": "(?i)^(anthropic\/)?(claude-3-7-sonnet-latest)$", + "createdAt": "2025-02-25T09:35:39.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cm7ka7zob000208jsfs9h5ajj_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cm7nusjvk0000tvmz71o85jwg", + "modelName": "gpt-4.5-preview", + "matchPattern": "(?i)^(openai\/)?(gpt-4.5-preview)$", + "createdAt": "2025-02-27T21:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7nusjvk0000tvmz71o85jwg_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000075, + "input_cached_tokens": 0.0000375, + "input_cached_text_tokens": 0.0000375, + "input_cache_read": 0.0000375, + "output": 0.00015 + } + } + ] + }, + { + "id": "cm7nusn640000tvmzf10z2x65", + "modelName": "gpt-4.5-preview-2025-02-27", + "matchPattern": "(?i)^(openai\/)?(gpt-4.5-preview-2025-02-27)$", + "createdAt": "2025-02-27T21:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7nusn640000tvmzf10z2x65_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000075, + "input_cached_tokens": 0.0000375, + "input_cached_text_tokens": 0.0000375, + "input_cache_read": 0.0000375, + "output": 0.00015 + } + } + ] + }, + { + "id": "cm7nusn643377tvmzh27m33kl", + "modelName": "gpt-4.1", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1)$", + "createdAt": "2025-04-15T10:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7nusn643377tvmzh27m33kl_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cached_text_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008 + } + } + ] + }, + { + "id": "cm7qahw732891bpmzy45r3x70", + "modelName": "gpt-4.1-2025-04-14", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1-2025-04-14)$", + "createdAt": "2025-04-15T10:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7qahw732891bpmzy45r3x70_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cached_text_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008 + } + } + ] + }, + { + "id": "cm7sglt825463kxnza72p6v81", + "modelName": "gpt-4.1-mini-2025-04-14", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1-mini-2025-04-14)$", + "createdAt": "2025-04-15T10:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7sglt825463kxnza72p6v81_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "input_cached_tokens": 1e-7, + "input_cached_text_tokens": 1e-7, + "input_cache_read": 1e-7, + "output": 0.0000016 + } + } + ] + }, + { + "id": "cm7vxpz967124dhjtb95w8f92", + "modelName": "gpt-4.1-nano-2025-04-14", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1-nano-2025-04-14)$", + "createdAt": "2025-04-15T10:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7vxpz967124dhjtb95w8f92_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_cached_tokens": 2.5e-8, + "input_cached_text_tokens": 2.5e-8, + "input_cache_read": 2.5e-8, + "output": 4e-7 + } + } + ] + }, + { + "id": "cm7wmny967124dhjtb95w8f81", + "modelName": "o3", + "matchPattern": "(?i)^(openai\/)?(o3)$", + "createdAt": "2025-04-16T23:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7wmny967124dhjtb95w8f81_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008, + "output_reasoning_tokens": 0.000008, + "output_reasoning": 0.000008 + } + } + ] + }, + { + "id": "cm7wopq3327124dhjtb95w8f81", + "modelName": "o3-2025-04-16", + "matchPattern": "(?i)^(openai\/)?(o3-2025-04-16)$", + "createdAt": "2025-04-16T23:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7wopq3327124dhjtb95w8f81_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008, + "output_reasoning_tokens": 0.000008, + "output_reasoning": 0.000008 + } + } + ] + }, + { + "id": "cm7wqrs1327124dhjtb95w8f81", + "modelName": "o4-mini", + "matchPattern": "(?i)^(o4-mini)$", + "createdAt": "2025-04-16T23:26:54.132Z", + "updatedAt": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "id": "cm7wqrs1327124dhjtb95w8f81_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 2.75e-7, + "input_cache_read": 2.75e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm7zqrs1327124dhjtb95w8f82", + "modelName": "o4-mini-2025-04-16", + "matchPattern": "(?i)^(o4-mini-2025-04-16)$", + "createdAt": "2025-04-16T23:26:54.132Z", + "updatedAt": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "id": "cm7zqrs1327124dhjtb95w8f82_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 2.75e-7, + "input_cache_read": 2.75e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "id": "cm7zsrs1327124dhjtb95w8f74", + "modelName": "gemini-2.0-flash", + "matchPattern": "(?i)^(google\/)?(gemini-2.0-flash)(@[a-zA-Z0-9]+)?$", + "createdAt": "2025-04-22T10:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7zsrs1327124dhjtb95w8f74_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "output": 4e-7 + } + } + ] + }, + { + "id": "cm7ztrs1327124dhjtb95w8f19", + "modelName": "gemini-2.0-flash-lite-preview", + "matchPattern": "(?i)^(google\/)?(gemini-2.0-flash-lite-preview)(@[a-zA-Z0-9]+)?$", + "createdAt": "2025-04-22T10:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cm7ztrs1327124dhjtb95w8f19_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-8, + "output": 3e-7 + } + } + ] + }, + { + "id": "cm7zxrs1327124dhjtb95w8f45", + "modelName": "gpt-4.1-nano", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1-nano)$", + "createdAt": "2025-04-22T10:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7zxrs1327124dhjtb95w8f45_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_cached_tokens": 2.5e-8, + "input_cached_text_tokens": 2.5e-8, + "input_cache_read": 2.5e-8, + "output": 4e-7 + } + } + ] + }, + { + "id": "cm7zzrs1327124dhjtb95w8p96", + "modelName": "gpt-4.1-mini", + "matchPattern": "(?i)^(openai\/)?(gpt-4.1-mini)$", + "createdAt": "2025-04-22T10:11:35.241Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cm7zzrs1327124dhjtb95w8p96_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "input_cached_tokens": 1e-7, + "input_cached_text_tokens": 1e-7, + "input_cache_read": 1e-7, + "output": 0.0000016 + } + } + ] + }, + { + "id": "c5qmrqolku82tra3vgdixmys", + "modelName": "claude-sonnet-4-5-20250929", + "matchPattern": "(?i)^(anthropic\/)?(claude-sonnet-4-5(-20250929)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-5(-20250929)?-v1(:0)?|claude-sonnet-4-5-V1(@20250929)?|claude-sonnet-4-5(@20250929)?)$", + "createdAt": "2025-09-29T00:00:00.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "c5qmrqolku82tra3vgdixmys_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + }, + { + "id": "00b65240-047b-4722-9590-808edbc2067f", + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "input", + "operator": "gt", + "value": 200000, + "caseSensitive": false + } + ], + "prices": { + "input": 0.000006, + "input_tokens": 0.000006, + "output": 0.0000225, + "output_tokens": 0.0000225, + "cache_creation_input_tokens": 0.0000075, + "input_cache_creation": 0.0000075, + "input_cache_creation_5m": 0.0000075, + "input_cache_creation_1h": 0.000012, + "cache_read_input_tokens": 6e-7, + "input_cache_read": 6e-7 + } + } + ] + }, + { + "id": "cmazmkzlm00000djp1e1qe4k4", + "modelName": "claude-sonnet-4-20250514", + "matchPattern": "(?i)^(anthropic\/)?(claude-sonnet-4(-20250514)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4(-20250514)?-v1(:0)?|claude-sonnet-4-V1(@20250514)?|claude-sonnet-4(@20250514)?)$", + "createdAt": "2025-05-22T17:09:02.131Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmazmkzlm00000djp1e1qe4k4_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cmazmlbnv00010djpazed91va", + "modelName": "claude-sonnet-4-latest", + "matchPattern": "(?i)^(anthropic\/)?(claude-sonnet-4-latest)$", + "createdAt": "2025-05-22T17:09:02.131Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmazmlbnv00010djpazed91va_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "id": "cmazmlm2p00020djpa9s64jw5", + "modelName": "claude-opus-4-20250514", + "matchPattern": "(?i)^(anthropic\/)?(claude-opus-4(-20250514)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4(-20250514)?-v1(:0)?|claude-opus-4(@20250514)?)$", + "createdAt": "2025-05-22T17:09:02.131Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmazmlm2p00020djpa9s64jw5_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_tokens": 0.000015, + "output": 0.000075, + "output_tokens": 0.000075, + "cache_creation_input_tokens": 0.00001875, + "input_cache_creation": 0.00001875, + "input_cache_creation_5m": 0.00001875, + "input_cache_creation_1h": 0.00003, + "cache_read_input_tokens": 0.0000015, + "input_cache_read": 0.0000015 + } + } + ] + }, + { + "id": "cmz9x72kq55721pqrs83y4n2bx", + "modelName": "o3-pro", + "matchPattern": "(?i)^(openai\/)?(o3-pro)$", + "createdAt": "2025-06-10T22:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cmz9x72kq55721pqrs83y4n2bx_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00002, + "output": 0.00008, + "output_reasoning_tokens": 0.00008, + "output_reasoning": 0.00008 + } + } + ] + }, + { + "id": "cmz9x72kq55721pqrs83y4n2by", + "modelName": "o3-pro-2025-06-10", + "matchPattern": "(?i)^(openai\/)?(o3-pro-2025-06-10)$", + "createdAt": "2025-06-10T22:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cmz9x72kq55721pqrs83y4n2by_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00002, + "output": 0.00008, + "output_reasoning_tokens": 0.00008, + "output_reasoning": 0.00008 + } + } + ] + }, + { + "id": "cmbrold5b000107lbftb9fdoo", + "modelName": "o1-pro", + "matchPattern": "(?i)^(openai\/)?(o1-pro)$", + "createdAt": "2025-06-10T22:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cmbrold5b000107lbftb9fdoo_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00015, + "output": 0.0006, + "output_reasoning_tokens": 0.0006, + "output_reasoning": 0.0006 + } + } + ] + }, + { + "id": "cmbrolpax000207lb3xkedysz", + "modelName": "o1-pro-2025-03-19", + "matchPattern": "(?i)^(openai\/)?(o1-pro-2025-03-19)$", + "createdAt": "2025-06-10T22:26:54.132Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cmbrolpax000207lb3xkedysz_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00015, + "output": 0.0006, + "output_reasoning_tokens": 0.0006, + "output_reasoning": 0.0006 + } + } + ] + }, + { + "id": "cmcnjkfwn000107l43bf5e8ax", + "modelName": "gemini-2.5-flash", + "matchPattern": "(?i)^(google\/)?(gemini-2.5-flash)$", + "createdAt": "2025-07-03T13:44:06.964Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "pricingTiers": [ + { + "id": "cmcnjkfwn000107l43bf5e8ax_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 3e-7, + "input_text": 3e-7, + "input_modality_1": 3e-7, + "prompt_token_count": 3e-7, + "promptTokenCount": 3e-7, + "input_cached_tokens": 3e-8, + "cached_content_token_count": 3e-8, + "output": 0.0000025, + "output_modality_1": 0.0000025, + "candidates_token_count": 0.0000025, + "candidatesTokenCount": 0.0000025, + "thoughtsTokenCount": 0.0000025, + "thoughts_token_count": 0.0000025, + "output_reasoning": 0.0000025, + "input_audio_tokens": 0.000001 + } + } + ] + }, + { + "id": "cmcnjkrfa000207l4fpnh5mnv", + "modelName": "gemini-2.5-flash-lite", + "matchPattern": "(?i)^(google\/)?(gemini-2.5-flash-lite)$", + "createdAt": "2025-07-03T13:44:06.964Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "pricingTiers": [ + { + "id": "cmcnjkrfa000207l4fpnh5mnv_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_text": 1e-7, + "input_modality_1": 1e-7, + "prompt_token_count": 1e-7, + "promptTokenCount": 1e-7, + "input_cached_tokens": 2.5e-8, + "cached_content_token_count": 2.5e-8, + "output": 4e-7, + "output_modality_1": 4e-7, + "candidates_token_count": 4e-7, + "candidatesTokenCount": 4e-7, + "thoughtsTokenCount": 4e-7, + "thoughts_token_count": 4e-7, + "output_reasoning": 4e-7, + "input_audio_tokens": 5e-7 + } + } + ] + }, + { + "id": "cmdysde5w0000rkmzbc1g5au3", + "modelName": "claude-opus-4-1-20250805", + "matchPattern": "(?i)^(anthropic\/)?(claude-opus-4-1(-20250805)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4-1(-20250805)?-v1(:0)?|claude-opus-4-1(@20250805)?)$", + "createdAt": "2025-08-05T15:00:00.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmdysde5w0000rkmzbc1g5au3_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_tokens": 0.000015, + "output": 0.000075, + "output_tokens": 0.000075, + "cache_creation_input_tokens": 0.00001875, + "input_cache_creation": 0.00001875, + "input_cache_creation_5m": 0.00001875, + "input_cache_creation_1h": 0.00003, + "cache_read_input_tokens": 0.0000015, + "input_cache_read": 0.0000015 + } + } + ] + }, + { + "id": "38c3822a-09a3-457b-b200-2c6f17f7cf2f", + "modelName": "gpt-5", + "matchPattern": "(?i)^(openai\/)?(gpt-5)$", + "createdAt": "2025-08-07T16:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "38c3822a-09a3-457b-b200-2c6f17f7cf2f_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "id": "12543803-2d5f-4189-addc-821ad71c8b55", + "modelName": "gpt-5-2025-08-07", + "matchPattern": "(?i)^(openai\/)?(gpt-5-2025-08-07)$", + "createdAt": "2025-08-11T08:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "12543803-2d5f-4189-addc-821ad71c8b55_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "id": "3d6a975a-a42d-4ea2-a3ec-4ae567d5a364", + "modelName": "gpt-5-mini", + "matchPattern": "(?i)^(openai\/)?(gpt-5-mini)$", + "createdAt": "2025-08-07T16:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "3d6a975a-a42d-4ea2-a3ec-4ae567d5a364_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "output": 0.000002, + "input_cache_read": 2.5e-8, + "output_reasoning_tokens": 0.000002, + "output_reasoning": 0.000002 + } + } + ] + }, + { + "id": "03b83894-7172-4e1e-8e8b-37d792484efd", + "modelName": "gpt-5-mini-2025-08-07", + "matchPattern": "(?i)^(openai\/)?(gpt-5-mini-2025-08-07)$", + "createdAt": "2025-08-11T08:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "03b83894-7172-4e1e-8e8b-37d792484efd_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "output": 0.000002, + "input_cache_read": 2.5e-8, + "output_reasoning_tokens": 0.000002, + "output_reasoning": 0.000002 + } + } + ] + }, + { + "id": "f0b40234-b694-4c40-9494-7b0efd860fb9", + "modelName": "gpt-5-nano", + "matchPattern": "(?i)^(openai\/)?(gpt-5-nano)$", + "createdAt": "2025-08-07T16:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "f0b40234-b694-4c40-9494-7b0efd860fb9_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-8, + "input_cached_tokens": 5e-9, + "output": 4e-7, + "input_cache_read": 5e-9, + "output_reasoning_tokens": 4e-7, + "output_reasoning": 4e-7 + } + } + ] + }, + { + "id": "4489fde4-a594-4011-948b-526989300cd3", + "modelName": "gpt-5-nano-2025-08-07", + "matchPattern": "(?i)^(openai\/)?(gpt-5-nano-2025-08-07)$", + "createdAt": "2025-08-11T08:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "4489fde4-a594-4011-948b-526989300cd3_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-8, + "input_cached_tokens": 5e-9, + "output": 4e-7, + "input_cache_read": 5e-9, + "output_reasoning_tokens": 4e-7, + "output_reasoning": 4e-7 + } + } + ] + }, + { + "id": "8ba72ee3-ebe8-4110-a614-bf81094447e5", + "modelName": "gpt-5-chat-latest", + "matchPattern": "(?i)^(openai\/)?(gpt-5-chat-latest)$", + "createdAt": "2025-08-07T16:00:00.000Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "8ba72ee3-ebe8-4110-a614-bf81094447e5_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "id": "cmgg9zco3000004l258um9xk8", + "modelName": "gpt-5-pro", + "matchPattern": "(?i)^(openai\/)?(gpt-5-pro)$", + "createdAt": "2025-10-07T08:03:54.727Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmgg9zco3000004l258um9xk8_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.00012, + "output_reasoning_tokens": 0.00012, + "output_reasoning": 0.00012 + } + } + ] + }, + { + "id": "cmgga0vh9000104l22qe4fes4", + "modelName": "gpt-5-pro-2025-10-06", + "matchPattern": "(?i)^(openai\/)?(gpt-5-pro-2025-10-06)$", + "createdAt": "2025-10-07T08:03:54.727Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmgga0vh9000104l22qe4fes4_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.00012, + "output_reasoning_tokens": 0.00012, + "output_reasoning": 0.00012 + } + } + ] + }, + { + "id": "cmgt5gnkv000104jx171tbq4e", + "modelName": "claude-haiku-4-5-20251001", + "matchPattern": "(?i)^(anthropic\/)?(claude-haiku-4-5-20251001|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-haiku-4-5-20251001-v1:0|claude-4-5-haiku@20251001)$", + "createdAt": "2025-10-16T08:20:44.558Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmgt5gnkv000104jx171tbq4e_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000001, + "input_tokens": 0.000001, + "output": 0.000005, + "output_tokens": 0.000005, + "cache_creation_input_tokens": 0.00000125, + "input_cache_creation": 0.00000125, + "input_cache_creation_5m": 0.00000125, + "input_cache_creation_1h": 0.000002, + "cache_read_input_tokens": 1e-7, + "input_cache_read": 1e-7 + } + } + ] + }, + { + "id": "cmhymgpym000d04ih34rndvhr", + "modelName": "gpt-5.1", + "matchPattern": "(?i)^(openai\/)?(gpt-5.1)$", + "createdAt": "2025-11-14T08:57:23.481Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmhymgpym000d04ih34rndvhr_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "id": "cmhymgxiw000e04ihh9pw12ef", + "modelName": "gpt-5.1-2025-11-13", + "matchPattern": "(?i)^(openai\/)?(gpt-5.1-2025-11-13)$", + "createdAt": "2025-11-14T08:57:23.481Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmhymgxiw000e04ihh9pw12ef_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "id": "cmieupdva000004l541kwae70", + "modelName": "claude-opus-4-5-20251101", + "matchPattern": "(?i)^(anthropic\/)?(claude-opus-4-5(-20251101)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-5(-20251101)?-v1(:0)?|claude-opus-4-5(@20251101)?)$", + "createdAt": "2025-11-24T20:53:27.571Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerConfig": null, + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "cmieupdva000004l541kwae70_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-6, + "input_tokens": 5e-6, + "output": 25e-6, + "output_tokens": 25e-6, + "cache_creation_input_tokens": 6.25e-6, + "input_cache_creation": 6.25e-6, + "input_cache_creation_5m": 6.25e-6, + "input_cache_creation_1h": 10e-6, + "cache_read_input_tokens": 0.5e-6, + "input_cache_read": 0.5e-6 + } + } + ] + }, + { + "id": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed", + "modelName": "claude-sonnet-4-6", + "matchPattern": "(?i)^(anthropic\\/)?(claude-sonnet-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-6(-v1(:0)?)?|claude-sonnet-4-6)$", + "createdAt": "2026-02-18T00:00:00.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerConfig": null, + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "90ec5ec3-1a48-4ff0-919c-70cdb8f632ed_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 3e-6, + "input_tokens": 3e-6, + "output": 15e-6, + "output_tokens": 15e-6, + "cache_creation_input_tokens": 3.75e-6, + "input_cache_creation": 3.75e-6, + "input_cache_creation_5m": 3.75e-6, + "input_cache_creation_1h": 6e-6, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + }, + { + "id": "7830bfc2-c464-4ffe-b9a2-6e741f6c5486", + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "input", + "operator": "gt", + "value": 200000, + "caseSensitive": false + } + ], + "prices": { + "input": 6e-6, + "input_tokens": 6e-6, + "output": 22.5e-6, + "output_tokens": 22.5e-6, + "cache_creation_input_tokens": 7.5e-6, + "input_cache_creation": 7.5e-6, + "input_cache_creation_5m": 7.5e-6, + "input_cache_creation_1h": 12e-6, + "cache_read_input_tokens": 6e-7, + "input_cache_read": 6e-7 + } + } + ] + }, + { + "id": "13458bc0-1c20-44c2-8753-172f54b67647", + "modelName": "claude-opus-4-6", + "matchPattern": "(?i)^(anthropic\/)?(claude-opus-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-6-v1(:0)?|claude-opus-4-6)$", + "createdAt": "2026-02-09T00:00:00.000Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "tokenizerConfig": null, + "tokenizerId": "claude", + "pricingTiers": [ + { + "id": "13458bc0-1c20-44c2-8753-172f54b67647_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-6, + "input_tokens": 5e-6, + "output": 25e-6, + "output_tokens": 25e-6, + "cache_creation_input_tokens": 6.25e-6, + "input_cache_creation": 6.25e-6, + "input_cache_creation_5m": 6.25e-6, + "input_cache_creation_1h": 10e-6, + "cache_read_input_tokens": 0.5e-6, + "input_cache_read": 0.5e-6 + } + } + ] + }, + { + "id": "cmig1hb7i000104l72qrzgc6h", + "modelName": "gemini-2.5-pro", + "matchPattern": "(?i)^(google\/)?(gemini-2.5-pro)$", + "createdAt": "2025-11-26T13:27:53.545Z", + "updatedAt": "2026-03-04T00:00:00.000Z", + "pricingTiers": [ + { + "id": "cmig1hb7i000104l72qrzgc6h_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-6, + "input_text": 1.25e-6, + "input_modality_1": 1.25e-6, + "prompt_token_count": 1.25e-6, + "promptTokenCount": 1.25e-6, + "input_cached_tokens": 0.125e-6, + "cached_content_token_count": 0.125e-6, + "output": 10e-6, + "output_modality_1": 10e-6, + "candidates_token_count": 10e-6, + "candidatesTokenCount": 10e-6, + "thoughtsTokenCount": 10e-6, + "thoughts_token_count": 10e-6, + "output_reasoning": 10e-6 + } + }, + { + "id": "bcf39e8f-9969-455f-be9a-541a00256092", + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000, + "caseSensitive": false + } + ], + "prices": { + "input": 2.5e-6, + "input_text": 2.5e-6, + "input_modality_1": 2.5e-6, + "prompt_token_count": 2.5e-6, + "promptTokenCount": 2.5e-6, + "input_cached_tokens": 0.25e-6, + "cached_content_token_count": 0.25e-6, + "output": 15e-6, + "output_modality_1": 15e-6, + "candidates_token_count": 15e-6, + "candidatesTokenCount": 15e-6, + "thoughtsTokenCount": 15e-6, + "thoughts_token_count": 15e-6, + "output_reasoning": 15e-6 + } + } + ] + }, + { + "id": "cmig1wmep000404l7fh6q5uog", + "modelName": "gemini-3-pro-preview", + "matchPattern": "(?i)^(google\/)?(gemini-3-pro-preview)$", + "createdAt": "2025-11-26T13:27:53.545Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "pricingTiers": [ + { + "id": "cmig1wmep000404l7fh6q5uog_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2e-6, + "input_modality_1": 2e-6, + "prompt_token_count": 2e-6, + "promptTokenCount": 2e-6, + "input_cached_tokens": 0.2e-6, + "cached_content_token_count": 0.2e-6, + "output": 12e-6, + "output_modality_1": 12e-6, + "candidates_token_count": 12e-6, + "candidatesTokenCount": 12e-6, + "thoughtsTokenCount": 12e-6, + "thoughts_token_count": 12e-6, + "output_reasoning": 12e-6 + } + }, + { + "id": "4da930c8-7146-4e27-b66c-b62f2c2ec357", + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000, + "caseSensitive": false + } + ], + "prices": { + "input": 4e-6, + "input_modality_1": 4e-6, + "prompt_token_count": 4e-6, + "promptTokenCount": 4e-6, + "input_cached_tokens": 0.4e-6, + "cached_content_token_count": 0.4e-6, + "output": 18e-6, + "output_modality_1": 18e-6, + "candidates_token_count": 18e-6, + "candidatesTokenCount": 18e-6, + "thoughtsTokenCount": 18e-6, + "thoughts_token_count": 18e-6, + "output_reasoning": 18e-6 + } + } + ] + }, + { + "id": "55106bba-a5dd-441b-bc0d-5652582b349d", + "modelName": "gemini-3.1-pro-preview", + "matchPattern": "(?i)^(google\/)?(gemini-3.1-pro-preview(-customtools)?)$", + "createdAt": "2026-02-19T00:00:00.000Z", + "updatedAt": "2026-02-19T00:00:00.000Z", + "pricingTiers": [ + { + "id": "55106bba-a5dd-441b-bc0d-5652582b349d_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2e-6, + "input_modality_1": 2e-6, + "prompt_token_count": 2e-6, + "promptTokenCount": 2e-6, + "input_cached_tokens": 0.2e-6, + "cached_content_token_count": 0.2e-6, + "output": 12e-6, + "output_modality_1": 12e-6, + "candidates_token_count": 12e-6, + "candidatesTokenCount": 12e-6, + "thoughtsTokenCount": 12e-6, + "thoughts_token_count": 12e-6, + "output_reasoning": 12e-6 + } + }, + { + "id": "ada11e9f-fe0d-465a-92af-ce334d0eedeb", + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000, + "caseSensitive": false + } + ], + "prices": { + "input": 4e-6, + "input_modality_1": 4e-6, + "prompt_token_count": 4e-6, + "promptTokenCount": 4e-6, + "input_cached_tokens": 0.4e-6, + "cached_content_token_count": 0.4e-6, + "output": 18e-6, + "output_modality_1": 18e-6, + "candidates_token_count": 18e-6, + "candidatesTokenCount": 18e-6, + "thoughtsTokenCount": 18e-6, + "thoughts_token_count": 18e-6, + "output_reasoning": 18e-6 + } + } + ] + }, + { + "id": "cmj2n4f2a000304kz49g4c43u", + "modelName": "gpt-5.2", + "matchPattern": "(?i)^(openai\/)?(gpt-5.2)$", + "createdAt": "2025-12-12T09:00:06.513Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmj2n4f2a000304kz49g4c43u_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.75e-6, + "input_cached_tokens": 0.175e-6, + "input_cache_read": 0.175e-6, + "output": 14e-6, + "output_reasoning_tokens": 14e-6, + "output_reasoning": 14e-6 + } + } + ] + }, + { + "id": "cmj2muxg6000104kzd2tc8953", + "modelName": "gpt-5.2-2025-12-11", + "matchPattern": "(?i)^(openai\/)?(gpt-5.2-2025-12-11)$", + "createdAt": "2025-12-12T09:00:06.513Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmj2muxg6000104kzd2tc8953_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.75e-6, + "input_cached_tokens": 0.175e-6, + "input_cache_read": 0.175e-6, + "output": 14e-6, + "output_reasoning_tokens": 14e-6, + "output_reasoning": 14e-6 + } + } + ] + }, + { + "id": "cmj2n6pkq000404kz2s0b6if7", + "modelName": "gpt-5.2-pro", + "matchPattern": "(?i)^(openai\/)?(gpt-5.2-pro)$", + "createdAt": "2025-12-12T09:00:06.513Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmj2n6pkq000404kz2s0b6if7_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 21e-6, + "output": 168e-6, + "output_reasoning_tokens": 168e-6, + "output_reasoning": 168e-6 + } + } + ] + }, + { + "id": "cmj2n70oe000504kz21b76mes", + "modelName": "gpt-5.2-pro-2025-12-11", + "matchPattern": "(?i)^(openai\/)?(gpt-5.2-pro-2025-12-11)$", + "createdAt": "2025-12-12T09:00:06.513Z", + "updatedAt": "2025-12-12T15:00:06.513Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "cmj2n70oe000504kz21b76mes_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 21e-6, + "output": 168e-6, + "output_reasoning_tokens": 168e-6, + "output_reasoning": 168e-6 + } + } + ] + }, + { + "id": "bee3c111-fe6f-4641-8775-73ea33b29fca", + "modelName": "gpt-5.4", + "matchPattern": "(?i)^(openai\/)?(gpt-5.4)$", + "createdAt": "2026-03-05T00:00:00.000Z", + "updatedAt": "2026-03-05T00:00:00.000Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "bee3c111-fe6f-4641-8775-73ea33b29fca_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-6, + "input_cached_tokens": 0.25e-6, + "input_cache_read": 0.25e-6, + "output": 15e-6, + "output_reasoning_tokens": 15e-6, + "output_reasoning": 15e-6 + } + } + ] + }, + { + "id": "68d32054-8748-4d25-9f64-d78d483601bd", + "modelName": "gpt-5.4-pro", + "matchPattern": "(?i)^(openai\/)?(gpt-5.4-pro)$", + "createdAt": "2026-03-05T00:00:00.000Z", + "updatedAt": "2026-03-05T00:00:00.000Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "68d32054-8748-4d25-9f64-d78d483601bd_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 30e-6, + "output": 180e-6, + "output_reasoning_tokens": 180e-6, + "output_reasoning": 180e-6 + } + } + ] + }, + { + "id": "22dfc7e1-1fe1-4286-b1af-928635e7ecb9", + "modelName": "gpt-5.4-2026-03-05", + "matchPattern": "(?i)^(openai\/)?(gpt-5.4-2026-03-05)$", + "createdAt": "2026-03-05T00:00:00.000Z", + "updatedAt": "2026-03-05T00:00:00.000Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "22dfc7e1-1fe1-4286-b1af-928635e7ecb9_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-6, + "input_cached_tokens": 0.25e-6, + "input_cache_read": 0.25e-6, + "output": 15e-6, + "output_reasoning_tokens": 15e-6, + "output_reasoning": 15e-6 + } + } + ] + }, + { + "id": "d8873413-05ab-4374-8223-e8c9005c4a0e", + "modelName": "gpt-5.4-pro-2026-03-05", + "matchPattern": "(?i)^(openai\/)?(gpt-5.4-pro-2026-03-05)$", + "createdAt": "2026-03-05T00:00:00.000Z", + "updatedAt": "2026-03-05T00:00:00.000Z", + "tokenizerConfig": { + "tokensPerName": 1, + "tokenizerModel": "gpt-4", + "tokensPerMessage": 3 + }, + "tokenizerId": "openai", + "pricingTiers": [ + { + "id": "d8873413-05ab-4374-8223-e8c9005c4a0e_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 30e-6, + "output": 180e-6, + "output_reasoning_tokens": 180e-6, + "output_reasoning": 180e-6 + } + } + ] + }, + { + "id": "cmjfoeykl000004l8ffzra8c7", + "modelName": "gemini-3-flash-preview", + "matchPattern": "(?i)^(google\/)?(gemini-3-flash-preview)$", + "createdAt": "2025-12-21T12:01:42.282Z", + "updatedAt": "2025-12-21T12:01:42.282Z", + "pricingTiers": [ + { + "id": "cmjfoeykl000004l8ffzra8c7_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.5e-6, + "input_modality_1": 0.5e-6, + "prompt_token_count": 0.5e-6, + "promptTokenCount": 0.5e-6, + "input_cached_tokens": 0.05e-6, + "cached_content_token_count": 0.05e-6, + "output": 3e-6, + "output_modality_1": 3e-6, + "candidates_token_count": 3e-6, + "candidatesTokenCount": 3e-6, + "thoughtsTokenCount": 3e-6, + "thoughts_token_count": 3e-6, + "output_reasoning": 3e-6 + } + } + ] + }, + { + "id": "0707bef8-10c8-46e2-a871-0436f05f0b92", + "modelName": "gemini-3.1-flash-lite-preview", + "matchPattern": "(?i)^(google\/)?(gemini-3.1-flash-lite-preview)$", + "createdAt": "2026-03-03T00:00:00.000Z", + "updatedAt": "2026-03-03T00:00:00.000Z", + "pricingTiers": [ + { + "id": "0707bef8-10c8-46e2-a871-0436f05f0b92_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.25e-6, + "input_modality_1": 0.25e-6, + "prompt_token_count": 0.25e-6, + "promptTokenCount": 0.25e-6, + "input_cached_tokens": 0.025e-6, + "cached_content_token_count": 0.025e-6, + "output": 1.5e-6, + "output_modality_1": 1.5e-6, + "candidates_token_count": 1.5e-6, + "candidatesTokenCount": 1.5e-6, + "thoughtsTokenCount": 1.5e-6, + "thoughts_token_count": 1.5e-6, + "output_reasoning": 1.5e-6, + "input_audio_tokens": 0.5e-6 + } + } + ] + } +] diff --git a/internal-packages/llm-pricing/src/defaultPrices.ts b/internal-packages/llm-pricing/src/defaultPrices.ts new file mode 100644 index 00000000000..91b8a6f89f6 --- /dev/null +++ b/internal-packages/llm-pricing/src/defaultPrices.ts @@ -0,0 +1,2984 @@ +import type { DefaultModelDefinition } from "./types.js"; + +// Auto-generated from Langfuse default-model-prices.json — do not edit manually. +// Run `pnpm run sync-prices` to update from upstream. +// Source: https://github.com/langfuse/langfuse + +export const defaultModelPrices: DefaultModelDefinition[] = [ + { + "modelName": "gpt-4o", + "matchPattern": "(?i)^(openai/)?(gpt-4o)$", + "startDate": "2024-05-13T23:15:07.670Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-4o-2024-05-13", + "matchPattern": "(?i)^(openai/)?(gpt-4o-2024-05-13)$", + "startDate": "2024-05-13T23:15:07.670Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "output": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-4-1106-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-1106-preview)$", + "startDate": "2024-04-23T10:37:17.092Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "gpt-4-turbo-vision", + "matchPattern": "(?i)^(openai/)?(gpt-4(-\\d{4})?-vision-preview)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "gpt-4-32k", + "matchPattern": "(?i)^(openai/)?(gpt-4-32k)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-4-32k-0613", + "matchPattern": "(?i)^(openai/)?(gpt-4-32k-0613)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-1106", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-1106)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000001, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-0613", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0613)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000015, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-4-0613", + "matchPattern": "(?i)^(openai/)?(gpt-4-0613)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-instruct", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-instruct)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000015, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "text-ada-001", + "matchPattern": "(?i)^(text-ada-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.000004 + } + } + ] + }, + { + "modelName": "text-babbage-001", + "matchPattern": "(?i)^(text-babbage-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 5e-7 + } + } + ] + }, + { + "modelName": "text-curie-001", + "matchPattern": "(?i)^(text-curie-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-davinci-001", + "matchPattern": "(?i)^(text-davinci-001)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-davinci-002", + "matchPattern": "(?i)^(text-davinci-002)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-davinci-003", + "matchPattern": "(?i)^(text-davinci-003)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 0.00002 + } + } + ] + }, + { + "modelName": "text-embedding-ada-002-v2", + "matchPattern": "(?i)^(text-embedding-ada-002-v2)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "text-embedding-ada-002", + "matchPattern": "(?i)^(text-embedding-ada-002)$", + "startDate": "2024-01-24T18:18:50.861Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-16k-0613", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-16k-0613)$", + "startDate": "2024-02-03T17:29:57.350Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "output": 0.000004 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-0301", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0301)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "output": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-4-32k-0314", + "matchPattern": "(?i)^(openai/)?(gpt-4-32k-0314)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00006, + "output": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-4-0314", + "matchPattern": "(?i)^(openai/)?(gpt-4-0314)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "modelName": "gpt-4", + "matchPattern": "(?i)^(openai/)?(gpt-4)$", + "startDate": "2024-01-24T10:19:21.693Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00006 + } + } + ] + }, + { + "modelName": "claude-instant-1.2", + "matchPattern": "(?i)^(anthropic/)?(claude-instant-1.2)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000163, + "output": 0.00000551 + } + } + ] + }, + { + "modelName": "claude-2.0", + "matchPattern": "(?i)^(anthropic/)?(claude-2.0)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-2.1", + "matchPattern": "(?i)^(anthropic/)?(claude-2.1)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-1.3", + "matchPattern": "(?i)^(anthropic/)?(claude-1.3)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-1.2", + "matchPattern": "(?i)^(anthropic/)?(claude-1.2)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-1.1", + "matchPattern": "(?i)^(anthropic/)?(claude-1.1)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000008, + "output": 0.000024 + } + } + ] + }, + { + "modelName": "claude-instant-1", + "matchPattern": "(?i)^(anthropic/)?(claude-instant-1)$", + "startDate": "2024-01-30T15:44:13.447Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000163, + "output": 0.00000551 + } + } + ] + }, + { + "modelName": "babbage-002", + "matchPattern": "(?i)^(babbage-002)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "davinci-002", + "matchPattern": "(?i)^(davinci-002)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000006, + "output": 0.000012 + } + } + ] + }, + { + "modelName": "text-embedding-3-small", + "matchPattern": "(?i)^(text-embedding-3-small)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 2e-8 + } + } + ] + }, + { + "modelName": "text-embedding-3-large", + "matchPattern": "(?i)^(text-embedding-3-large)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1.3e-7 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-0125", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-0125)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo)$", + "startDate": "2024-02-13T12:00:37.424Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-4-0125-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-0125-preview)$", + "startDate": "2024-01-26T17:35:21.129Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "ft:gpt-3.5-turbo-1106", + "matchPattern": "(?i)^(ft:)(gpt-3.5-turbo-1106:)(.+)(:)(.*)(:)(.+)$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "output": 0.000006 + } + } + ] + }, + { + "modelName": "ft:gpt-3.5-turbo-0613", + "matchPattern": "(?i)^(ft:)(gpt-3.5-turbo-0613:)(.+)(:)(.*)(:)(.+)$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000012, + "output": 0.000016 + } + } + ] + }, + { + "modelName": "ft:davinci-002", + "matchPattern": "(?i)^(ft:)(davinci-002:)(.+)(:)(.*)(:)(.+)$$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000012, + "output": 0.000012 + } + } + ] + }, + { + "modelName": "ft:babbage-002", + "matchPattern": "(?i)^(ft:)(babbage-002:)(.+)(:)(.*)(:)(.+)$$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000016, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "chat-bison", + "matchPattern": "(?i)^(chat-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "codechat-bison-32k", + "matchPattern": "(?i)^(codechat-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "codechat-bison", + "matchPattern": "(?i)^(codechat-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "text-bison-32k", + "matchPattern": "(?i)^(text-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "chat-bison-32k", + "matchPattern": "(?i)^(chat-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "text-unicorn", + "matchPattern": "(?i)^(text-unicorn)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "output": 0.0000075 + } + } + ] + }, + { + "modelName": "text-bison", + "matchPattern": "(?i)^(text-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "textembedding-gecko", + "matchPattern": "(?i)^(textembedding-gecko)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "textembedding-gecko-multilingual", + "matchPattern": "(?i)^(textembedding-gecko-multilingual)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "total": 1e-7 + } + } + ] + }, + { + "modelName": "code-gecko", + "matchPattern": "(?i)^(code-gecko)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "code-bison", + "matchPattern": "(?i)^(code-bison)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "code-bison-32k", + "matchPattern": "(?i)^(code-bison-32k)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-01-31T13:25:02.141Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "gpt-3.5-turbo-16k", + "matchPattern": "(?i)^(openai/)?(gpt-)(35|3.5)(-turbo-16k)$", + "startDate": "2024-02-13T12:00:37.424Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "output": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-4-turbo-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-turbo-preview)$", + "startDate": "2024-02-15T21:21:50.947Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "claude-3-opus-20240229", + "matchPattern": "(?i)^(anthropic/)?(claude-3-opus-20240229|anthropic\\.claude-3-opus-20240229-v1:0|claude-3-opus@20240229)$", + "startDate": "2024-03-07T17:55:38.139Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.000075 + } + } + ] + }, + { + "modelName": "claude-3-sonnet-20240229", + "matchPattern": "(?i)^(anthropic/)?(claude-3-sonnet-20240229|anthropic\\.claude-3-sonnet-20240229-v1:0|claude-3-sonnet@20240229)$", + "startDate": "2024-03-07T17:55:38.139Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3-haiku-20240307", + "matchPattern": "(?i)^(anthropic/)?(claude-3-haiku-20240307|anthropic\\.claude-3-haiku-20240307-v1:0|claude-3-haiku@20240307)$", + "startDate": "2024-03-14T09:41:18.736Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 0.00000125 + } + } + ] + }, + { + "modelName": "gemini-1.0-pro-latest", + "matchPattern": "(?i)^(google/)?(gemini-1.0-pro-latest)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "output": 5e-7 + } + } + ] + }, + { + "modelName": "gemini-1.0-pro", + "matchPattern": "(?i)^(google/)?(gemini-1.0-pro)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "modelName": "gemini-1.0-pro-001", + "matchPattern": "(?i)^(google/)?(gemini-1.0-pro-001)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "modelName": "gemini-pro", + "matchPattern": "(?i)^(google/)?(gemini-pro)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.25e-7, + "output": 3.75e-7 + } + } + ] + }, + { + "modelName": "gemini-1.5-pro-latest", + "matchPattern": "(?i)^(google/)?(gemini-1.5-pro-latest)(@[a-zA-Z0-9]+)?$", + "startDate": "2024-04-11T10:27:46.517Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "output": 0.0000075 + } + } + ] + }, + { + "modelName": "gpt-4-turbo-2024-04-09", + "matchPattern": "(?i)^(openai/)?(gpt-4-turbo-2024-04-09)$", + "startDate": "2024-04-23T10:37:17.092Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "gpt-4-turbo", + "matchPattern": "(?i)^(openai/)?(gpt-4-turbo)$", + "startDate": "2024-04-11T21:13:44.989Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": " gpt-4-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4-preview)$", + "startDate": "2024-04-23T10:37:17.092Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00001, + "output": 0.00003 + } + } + ] + }, + { + "modelName": "claude-3-5-sonnet-20240620", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-sonnet-20240620|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20240620-v1:0|claude-3-5-sonnet@20240620)$", + "startDate": "2024-06-25T11:47:24.475Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "gpt-4o-mini", + "matchPattern": "(?i)^(openai/)?(gpt-4o-mini)$", + "startDate": "2024-07-18T17:56:09.591Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.5e-7, + "output": 6e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8 + } + } + ] + }, + { + "modelName": "gpt-4o-mini-2024-07-18", + "matchPattern": "(?i)^(openai/)?(gpt-4o-mini-2024-07-18)$", + "startDate": "2024-07-18T17:56:09.591Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1.5e-7, + "input_cached_tokens": 7.5e-8, + "input_cache_read": 7.5e-8, + "output": 6e-7 + } + } + ] + }, + { + "modelName": "gpt-4o-2024-08-06", + "matchPattern": "(?i)^(openai/)?(gpt-4o-2024-08-06)$", + "startDate": "2024-08-07T11:54:31.298Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "modelName": "o1-preview", + "matchPattern": "(?i)^(openai/)?(o1-preview)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o1-preview-2024-09-12", + "matchPattern": "(?i)^(openai/)?(o1-preview-2024-09-12)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o1-mini", + "matchPattern": "(?i)^(openai/)?(o1-mini)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "o1-mini-2024-09-12", + "matchPattern": "(?i)^(openai/)?(o1-mini-2024-09-12)$", + "startDate": "2024-09-13T10:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "claude-3.5-sonnet-20241022", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-sonnet-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-sonnet-20241022-v2:0|claude-3-5-sonnet-V2@20241022)$", + "startDate": "2024-10-22T18:48:01.676Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3.5-sonnet-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-sonnet-latest)$", + "startDate": "2024-10-22T18:48:01.676Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3-5-haiku-20241022", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-haiku-20241022|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3-5-haiku-20241022-v1:0|claude-3-5-haiku-V1@20241022)$", + "startDate": "2024-11-05T10:30:50.566Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 8e-7, + "input_tokens": 8e-7, + "output": 0.000004, + "output_tokens": 0.000004, + "cache_creation_input_tokens": 0.000001, + "input_cache_creation": 0.000001, + "input_cache_creation_5m": 0.000001, + "input_cache_creation_1h": 0.0000016, + "cache_read_input_tokens": 8e-8, + "input_cache_read": 8e-8 + } + } + ] + }, + { + "modelName": "claude-3.5-haiku-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-3-5-haiku-latest)$", + "startDate": "2024-11-05T10:30:50.566Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 8e-7, + "input_tokens": 8e-7, + "output": 0.000004, + "output_tokens": 0.000004, + "cache_creation_input_tokens": 0.000001, + "input_cache_creation": 0.000001, + "input_cache_creation_5m": 0.000001, + "input_cache_creation_1h": 0.0000016, + "cache_read_input_tokens": 8e-8, + "input_cache_read": 8e-8 + } + } + ] + }, + { + "modelName": "chatgpt-4o-latest", + "matchPattern": "(?i)^(chatgpt-4o-latest)$", + "startDate": "2024-11-25T12:47:17.504Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "output": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-4o-2024-11-20", + "matchPattern": "(?i)^(openai/)?(gpt-4o-2024-11-20)$", + "startDate": "2024-12-03T10:06:12.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 0.00000125, + "input_cache_read": 0.00000125, + "output": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-4o-audio-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4o-audio-preview)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.0000025, + "output_text_tokens": 0.00001, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "gpt-4o-audio-preview-2024-10-01", + "matchPattern": "(?i)^(openai/)?(gpt-4o-audio-preview-2024-10-01)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.0000025, + "output_text_tokens": 0.00001, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "gpt-4o-realtime-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4o-realtime-preview)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.000005, + "input_cached_text_tokens": 0.0000025, + "output_text_tokens": 0.00002, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "input_cached_audio_tokens": 0.00002, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "gpt-4o-realtime-preview-2024-10-01", + "matchPattern": "(?i)^(openai/)?(gpt-4o-realtime-preview-2024-10-01)$", + "startDate": "2024-12-03T10:19:56.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text_tokens": 0.000005, + "input_cached_text_tokens": 0.0000025, + "output_text_tokens": 0.00002, + "input_audio_tokens": 0.0001, + "input_audio": 0.0001, + "input_cached_audio_tokens": 0.00002, + "output_audio_tokens": 0.0002, + "output_audio": 0.0002 + } + } + ] + }, + { + "modelName": "o1", + "matchPattern": "(?i)^(openai/)?(o1)$", + "startDate": "2025-01-17T00:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o1-2024-12-17", + "matchPattern": "(?i)^(openai/)?(o1-2024-12-17)$", + "startDate": "2025-01-17T00:01:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_cached_tokens": 0.0000075, + "input_cache_read": 0.0000075, + "output": 0.00006, + "output_reasoning_tokens": 0.00006, + "output_reasoning": 0.00006 + } + } + ] + }, + { + "modelName": "o3-mini", + "matchPattern": "(?i)^(openai/)?(o3-mini)$", + "startDate": "2025-01-31T20:41:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "o3-mini-2025-01-31", + "matchPattern": "(?i)^(openai/)?(o3-mini-2025-01-31)$", + "startDate": "2025-01-31T20:41:35.373Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 5.5e-7, + "input_cache_read": 5.5e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash-001", + "matchPattern": "(?i)^(google/)?(gemini-2.0-flash-001)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-02-06T11:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash-lite-preview-02-05", + "matchPattern": "(?i)^(google/)?(gemini-2.0-flash-lite-preview-02-05)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-02-06T11:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-8, + "output": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3.7-sonnet-20250219", + "matchPattern": "(?i)^(anthropic/)?(claude-3.7-sonnet-20250219|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-3.7-sonnet-20250219-v1:0|claude-3-7-sonnet-V1@20250219)$", + "startDate": "2025-02-25T09:35:39.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-3.7-sonnet-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-3-7-sonnet-latest)$", + "startDate": "2025-02-25T09:35:39.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "gpt-4.5-preview", + "matchPattern": "(?i)^(openai/)?(gpt-4.5-preview)$", + "startDate": "2025-02-27T21:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000075, + "input_cached_tokens": 0.0000375, + "input_cached_text_tokens": 0.0000375, + "input_cache_read": 0.0000375, + "output": 0.00015 + } + } + ] + }, + { + "modelName": "gpt-4.5-preview-2025-02-27", + "matchPattern": "(?i)^(openai/)?(gpt-4.5-preview-2025-02-27)$", + "startDate": "2025-02-27T21:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000075, + "input_cached_tokens": 0.0000375, + "input_cached_text_tokens": 0.0000375, + "input_cache_read": 0.0000375, + "output": 0.00015 + } + } + ] + }, + { + "modelName": "gpt-4.1", + "matchPattern": "(?i)^(openai/)?(gpt-4.1)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cached_text_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008 + } + } + ] + }, + { + "modelName": "gpt-4.1-2025-04-14", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-2025-04-14)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cached_text_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008 + } + } + ] + }, + { + "modelName": "gpt-4.1-mini-2025-04-14", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-mini-2025-04-14)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "input_cached_tokens": 1e-7, + "input_cached_text_tokens": 1e-7, + "input_cache_read": 1e-7, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "gpt-4.1-nano-2025-04-14", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-nano-2025-04-14)$", + "startDate": "2025-04-15T10:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_cached_tokens": 2.5e-8, + "input_cached_text_tokens": 2.5e-8, + "input_cache_read": 2.5e-8, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "o3", + "matchPattern": "(?i)^(openai/)?(o3)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008, + "output_reasoning_tokens": 0.000008, + "output_reasoning": 0.000008 + } + } + ] + }, + { + "modelName": "o3-2025-04-16", + "matchPattern": "(?i)^(openai/)?(o3-2025-04-16)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_cached_tokens": 5e-7, + "input_cache_read": 5e-7, + "output": 0.000008, + "output_reasoning_tokens": 0.000008, + "output_reasoning": 0.000008 + } + } + ] + }, + { + "modelName": "o4-mini", + "matchPattern": "(?i)^(o4-mini)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 2.75e-7, + "input_cache_read": 2.75e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "o4-mini-2025-04-16", + "matchPattern": "(?i)^(o4-mini-2025-04-16)$", + "startDate": "2025-04-16T23:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000011, + "input_cached_tokens": 2.75e-7, + "input_cache_read": 2.75e-7, + "output": 0.0000044, + "output_reasoning_tokens": 0.0000044, + "output_reasoning": 0.0000044 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash", + "matchPattern": "(?i)^(google/)?(gemini-2.0-flash)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "gemini-2.0-flash-lite-preview", + "matchPattern": "(?i)^(google/)?(gemini-2.0-flash-lite-preview)(@[a-zA-Z0-9]+)?$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 7.5e-8, + "output": 3e-7 + } + } + ] + }, + { + "modelName": "gpt-4.1-nano", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-nano)$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_cached_tokens": 2.5e-8, + "input_cached_text_tokens": 2.5e-8, + "input_cache_read": 2.5e-8, + "output": 4e-7 + } + } + ] + }, + { + "modelName": "gpt-4.1-mini", + "matchPattern": "(?i)^(openai/)?(gpt-4.1-mini)$", + "startDate": "2025-04-22T10:11:35.241Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 4e-7, + "input_cached_tokens": 1e-7, + "input_cached_text_tokens": 1e-7, + "input_cache_read": 1e-7, + "output": 0.0000016 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-5-20250929", + "matchPattern": "(?i)^(anthropic/)?(claude-sonnet-4-5(-20250929)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-5(-20250929)?-v1(:0)?|claude-sonnet-4-5-V1(@20250929)?|claude-sonnet-4-5(@20250929)?)$", + "startDate": "2025-09-29T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "input", + "operator": "gt", + "value": 200000 + } + ], + "prices": { + "input": 0.000006, + "input_tokens": 0.000006, + "output": 0.0000225, + "output_tokens": 0.0000225, + "cache_creation_input_tokens": 0.0000075, + "input_cache_creation": 0.0000075, + "input_cache_creation_5m": 0.0000075, + "input_cache_creation_1h": 0.000012, + "cache_read_input_tokens": 6e-7, + "input_cache_read": 6e-7 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-20250514", + "matchPattern": "(?i)^(anthropic/)?(claude-sonnet-4(-20250514)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4(-20250514)?-v1(:0)?|claude-sonnet-4-V1(@20250514)?|claude-sonnet-4(@20250514)?)$", + "startDate": "2025-05-22T17:09:02.131Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-latest", + "matchPattern": "(?i)^(anthropic/)?(claude-sonnet-4-latest)$", + "startDate": "2025-05-22T17:09:02.131Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + } + ] + }, + { + "modelName": "claude-opus-4-20250514", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4(-20250514)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4(-20250514)?-v1(:0)?|claude-opus-4(@20250514)?)$", + "startDate": "2025-05-22T17:09:02.131Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_tokens": 0.000015, + "output": 0.000075, + "output_tokens": 0.000075, + "cache_creation_input_tokens": 0.00001875, + "input_cache_creation": 0.00001875, + "input_cache_creation_5m": 0.00001875, + "input_cache_creation_1h": 0.00003, + "cache_read_input_tokens": 0.0000015, + "input_cache_read": 0.0000015 + } + } + ] + }, + { + "modelName": "o3-pro", + "matchPattern": "(?i)^(openai/)?(o3-pro)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00002, + "output": 0.00008, + "output_reasoning_tokens": 0.00008, + "output_reasoning": 0.00008 + } + } + ] + }, + { + "modelName": "o3-pro-2025-06-10", + "matchPattern": "(?i)^(openai/)?(o3-pro-2025-06-10)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00002, + "output": 0.00008, + "output_reasoning_tokens": 0.00008, + "output_reasoning": 0.00008 + } + } + ] + }, + { + "modelName": "o1-pro", + "matchPattern": "(?i)^(openai/)?(o1-pro)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00015, + "output": 0.0006, + "output_reasoning_tokens": 0.0006, + "output_reasoning": 0.0006 + } + } + ] + }, + { + "modelName": "o1-pro-2025-03-19", + "matchPattern": "(?i)^(openai/)?(o1-pro-2025-03-19)$", + "startDate": "2025-06-10T22:26:54.132Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00015, + "output": 0.0006, + "output_reasoning_tokens": 0.0006, + "output_reasoning": 0.0006 + } + } + ] + }, + { + "modelName": "gemini-2.5-flash", + "matchPattern": "(?i)^(google/)?(gemini-2.5-flash)$", + "startDate": "2025-07-03T13:44:06.964Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 3e-7, + "input_text": 3e-7, + "input_modality_1": 3e-7, + "prompt_token_count": 3e-7, + "promptTokenCount": 3e-7, + "input_cached_tokens": 3e-8, + "cached_content_token_count": 3e-8, + "output": 0.0000025, + "output_modality_1": 0.0000025, + "candidates_token_count": 0.0000025, + "candidatesTokenCount": 0.0000025, + "thoughtsTokenCount": 0.0000025, + "thoughts_token_count": 0.0000025, + "output_reasoning": 0.0000025, + "input_audio_tokens": 0.000001 + } + } + ] + }, + { + "modelName": "gemini-2.5-flash-lite", + "matchPattern": "(?i)^(google/)?(gemini-2.5-flash-lite)$", + "startDate": "2025-07-03T13:44:06.964Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 1e-7, + "input_text": 1e-7, + "input_modality_1": 1e-7, + "prompt_token_count": 1e-7, + "promptTokenCount": 1e-7, + "input_cached_tokens": 2.5e-8, + "cached_content_token_count": 2.5e-8, + "output": 4e-7, + "output_modality_1": 4e-7, + "candidates_token_count": 4e-7, + "candidatesTokenCount": 4e-7, + "thoughtsTokenCount": 4e-7, + "thoughts_token_count": 4e-7, + "output_reasoning": 4e-7, + "input_audio_tokens": 5e-7 + } + } + ] + }, + { + "modelName": "claude-opus-4-1-20250805", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4-1(-20250805)?|(eu\\.|us\\.|apac\\.)?anthropic\\.claude-opus-4-1(-20250805)?-v1(:0)?|claude-opus-4-1(@20250805)?)$", + "startDate": "2025-08-05T15:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "input_tokens": 0.000015, + "output": 0.000075, + "output_tokens": 0.000075, + "cache_creation_input_tokens": 0.00001875, + "input_cache_creation": 0.00001875, + "input_cache_creation_5m": 0.00001875, + "input_cache_creation_1h": 0.00003, + "cache_read_input_tokens": 0.0000015, + "input_cache_read": 0.0000015 + } + } + ] + }, + { + "modelName": "gpt-5", + "matchPattern": "(?i)^(openai/)?(gpt-5)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5-2025-08-07", + "matchPattern": "(?i)^(openai/)?(gpt-5-2025-08-07)$", + "startDate": "2025-08-11T08:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5-mini", + "matchPattern": "(?i)^(openai/)?(gpt-5-mini)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "output": 0.000002, + "input_cache_read": 2.5e-8, + "output_reasoning_tokens": 0.000002, + "output_reasoning": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-5-mini-2025-08-07", + "matchPattern": "(?i)^(openai/)?(gpt-5-mini-2025-08-07)$", + "startDate": "2025-08-11T08:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "output": 0.000002, + "input_cache_read": 2.5e-8, + "output_reasoning_tokens": 0.000002, + "output_reasoning": 0.000002 + } + } + ] + }, + { + "modelName": "gpt-5-nano", + "matchPattern": "(?i)^(openai/)?(gpt-5-nano)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-8, + "input_cached_tokens": 5e-9, + "output": 4e-7, + "input_cache_read": 5e-9, + "output_reasoning_tokens": 4e-7, + "output_reasoning": 4e-7 + } + } + ] + }, + { + "modelName": "gpt-5-nano-2025-08-07", + "matchPattern": "(?i)^(openai/)?(gpt-5-nano-2025-08-07)$", + "startDate": "2025-08-11T08:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-8, + "input_cached_tokens": 5e-9, + "output": 4e-7, + "input_cache_read": 5e-9, + "output_reasoning_tokens": 4e-7, + "output_reasoning": 4e-7 + } + } + ] + }, + { + "modelName": "gpt-5-chat-latest", + "matchPattern": "(?i)^(openai/)?(gpt-5-chat-latest)$", + "startDate": "2025-08-07T16:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5-pro", + "matchPattern": "(?i)^(openai/)?(gpt-5-pro)$", + "startDate": "2025-10-07T08:03:54.727Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.00012, + "output_reasoning_tokens": 0.00012, + "output_reasoning": 0.00012 + } + } + ] + }, + { + "modelName": "gpt-5-pro-2025-10-06", + "matchPattern": "(?i)^(openai/)?(gpt-5-pro-2025-10-06)$", + "startDate": "2025-10-07T08:03:54.727Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000015, + "output": 0.00012, + "output_reasoning_tokens": 0.00012, + "output_reasoning": 0.00012 + } + } + ] + }, + { + "modelName": "claude-haiku-4-5-20251001", + "matchPattern": "(?i)^(anthropic/)?(claude-haiku-4-5-20251001|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-haiku-4-5-20251001-v1:0|claude-4-5-haiku@20251001)$", + "startDate": "2025-10-16T08:20:44.558Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000001, + "input_tokens": 0.000001, + "output": 0.000005, + "output_tokens": 0.000005, + "cache_creation_input_tokens": 0.00000125, + "input_cache_creation": 0.00000125, + "input_cache_creation_5m": 0.00000125, + "input_cache_creation_1h": 0.000002, + "cache_read_input_tokens": 1e-7, + "input_cache_read": 1e-7 + } + } + ] + }, + { + "modelName": "gpt-5.1", + "matchPattern": "(?i)^(openai/)?(gpt-5.1)$", + "startDate": "2025-11-14T08:57:23.481Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "gpt-5.1-2025-11-13", + "matchPattern": "(?i)^(openai/)?(gpt-5.1-2025-11-13)$", + "startDate": "2025-11-14T08:57:23.481Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_cached_tokens": 1.25e-7, + "output": 0.00001, + "input_cache_read": 1.25e-7, + "output_reasoning_tokens": 0.00001, + "output_reasoning": 0.00001 + } + } + ] + }, + { + "modelName": "claude-opus-4-5-20251101", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4-5(-20251101)?|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-5(-20251101)?-v1(:0)?|claude-opus-4-5(@20251101)?)$", + "startDate": "2025-11-24T20:53:27.571Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "input_tokens": 0.000005, + "output": 0.000025, + "output_tokens": 0.000025, + "cache_creation_input_tokens": 0.00000625, + "input_cache_creation": 0.00000625, + "input_cache_creation_5m": 0.00000625, + "input_cache_creation_1h": 0.00001, + "cache_read_input_tokens": 5e-7, + "input_cache_read": 5e-7 + } + } + ] + }, + { + "modelName": "claude-sonnet-4-6", + "matchPattern": "(?i)^(anthropic\\/)?(claude-sonnet-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-sonnet-4-6(-v1(:0)?)?|claude-sonnet-4-6)$", + "startDate": "2026-02-18T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000003, + "input_tokens": 0.000003, + "output": 0.000015, + "output_tokens": 0.000015, + "cache_creation_input_tokens": 0.00000375, + "input_cache_creation": 0.00000375, + "input_cache_creation_5m": 0.00000375, + "input_cache_creation_1h": 0.000006, + "cache_read_input_tokens": 3e-7, + "input_cache_read": 3e-7 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "input", + "operator": "gt", + "value": 200000 + } + ], + "prices": { + "input": 0.000006, + "input_tokens": 0.000006, + "output": 0.0000225, + "output_tokens": 0.0000225, + "cache_creation_input_tokens": 0.0000075, + "input_cache_creation": 0.0000075, + "input_cache_creation_5m": 0.0000075, + "input_cache_creation_1h": 0.000012, + "cache_read_input_tokens": 6e-7, + "input_cache_read": 6e-7 + } + } + ] + }, + { + "modelName": "claude-opus-4-6", + "matchPattern": "(?i)^(anthropic/)?(claude-opus-4-6|(eu\\.|us\\.|apac\\.|global\\.)?anthropic\\.claude-opus-4-6-v1(:0)?|claude-opus-4-6)$", + "startDate": "2026-02-09T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000005, + "input_tokens": 0.000005, + "output": 0.000025, + "output_tokens": 0.000025, + "cache_creation_input_tokens": 0.00000625, + "input_cache_creation": 0.00000625, + "input_cache_creation_5m": 0.00000625, + "input_cache_creation_1h": 0.00001, + "cache_read_input_tokens": 5e-7, + "input_cache_read": 5e-7 + } + } + ] + }, + { + "modelName": "gemini-2.5-pro", + "matchPattern": "(?i)^(google/)?(gemini-2.5-pro)$", + "startDate": "2025-11-26T13:27:53.545Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000125, + "input_text": 0.00000125, + "input_modality_1": 0.00000125, + "prompt_token_count": 0.00000125, + "promptTokenCount": 0.00000125, + "input_cached_tokens": 1.25e-7, + "cached_content_token_count": 1.25e-7, + "output": 0.00001, + "output_modality_1": 0.00001, + "candidates_token_count": 0.00001, + "candidatesTokenCount": 0.00001, + "thoughtsTokenCount": 0.00001, + "thoughts_token_count": 0.00001, + "output_reasoning": 0.00001 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000 + } + ], + "prices": { + "input": 0.0000025, + "input_text": 0.0000025, + "input_modality_1": 0.0000025, + "prompt_token_count": 0.0000025, + "promptTokenCount": 0.0000025, + "input_cached_tokens": 2.5e-7, + "cached_content_token_count": 2.5e-7, + "output": 0.000015, + "output_modality_1": 0.000015, + "candidates_token_count": 0.000015, + "candidatesTokenCount": 0.000015, + "thoughtsTokenCount": 0.000015, + "thoughts_token_count": 0.000015, + "output_reasoning": 0.000015 + } + } + ] + }, + { + "modelName": "gemini-3-pro-preview", + "matchPattern": "(?i)^(google/)?(gemini-3-pro-preview)$", + "startDate": "2025-11-26T13:27:53.545Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_modality_1": 0.000002, + "prompt_token_count": 0.000002, + "promptTokenCount": 0.000002, + "input_cached_tokens": 2e-7, + "cached_content_token_count": 2e-7, + "output": 0.000012, + "output_modality_1": 0.000012, + "candidates_token_count": 0.000012, + "candidatesTokenCount": 0.000012, + "thoughtsTokenCount": 0.000012, + "thoughts_token_count": 0.000012, + "output_reasoning": 0.000012 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000 + } + ], + "prices": { + "input": 0.000004, + "input_modality_1": 0.000004, + "prompt_token_count": 0.000004, + "promptTokenCount": 0.000004, + "input_cached_tokens": 4e-7, + "cached_content_token_count": 4e-7, + "output": 0.000018, + "output_modality_1": 0.000018, + "candidates_token_count": 0.000018, + "candidatesTokenCount": 0.000018, + "thoughtsTokenCount": 0.000018, + "thoughts_token_count": 0.000018, + "output_reasoning": 0.000018 + } + } + ] + }, + { + "modelName": "gemini-3.1-pro-preview", + "matchPattern": "(?i)^(google/)?(gemini-3.1-pro-preview(-customtools)?)$", + "startDate": "2026-02-19T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000002, + "input_modality_1": 0.000002, + "prompt_token_count": 0.000002, + "promptTokenCount": 0.000002, + "input_cached_tokens": 2e-7, + "cached_content_token_count": 2e-7, + "output": 0.000012, + "output_modality_1": 0.000012, + "candidates_token_count": 0.000012, + "candidatesTokenCount": 0.000012, + "thoughtsTokenCount": 0.000012, + "thoughts_token_count": 0.000012, + "output_reasoning": 0.000012 + } + }, + { + "name": "Large Context", + "isDefault": false, + "priority": 1, + "conditions": [ + { + "usageDetailPattern": "(input|prompt|cached)", + "operator": "gt", + "value": 200000 + } + ], + "prices": { + "input": 0.000004, + "input_modality_1": 0.000004, + "prompt_token_count": 0.000004, + "promptTokenCount": 0.000004, + "input_cached_tokens": 4e-7, + "cached_content_token_count": 4e-7, + "output": 0.000018, + "output_modality_1": 0.000018, + "candidates_token_count": 0.000018, + "candidatesTokenCount": 0.000018, + "thoughtsTokenCount": 0.000018, + "thoughts_token_count": 0.000018, + "output_reasoning": 0.000018 + } + } + ] + }, + { + "modelName": "gpt-5.2", + "matchPattern": "(?i)^(openai/)?(gpt-5.2)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000175, + "input_cached_tokens": 1.75e-7, + "input_cache_read": 1.75e-7, + "output": 0.000014, + "output_reasoning_tokens": 0.000014, + "output_reasoning": 0.000014 + } + } + ] + }, + { + "modelName": "gpt-5.2-2025-12-11", + "matchPattern": "(?i)^(openai/)?(gpt-5.2-2025-12-11)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00000175, + "input_cached_tokens": 1.75e-7, + "input_cache_read": 1.75e-7, + "output": 0.000014, + "output_reasoning_tokens": 0.000014, + "output_reasoning": 0.000014 + } + } + ] + }, + { + "modelName": "gpt-5.2-pro", + "matchPattern": "(?i)^(openai/)?(gpt-5.2-pro)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000021, + "output": 0.000168, + "output_reasoning_tokens": 0.000168, + "output_reasoning": 0.000168 + } + } + ] + }, + { + "modelName": "gpt-5.2-pro-2025-12-11", + "matchPattern": "(?i)^(openai/)?(gpt-5.2-pro-2025-12-11)$", + "startDate": "2025-12-12T09:00:06.513Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.000021, + "output": 0.000168, + "output_reasoning_tokens": 0.000168, + "output_reasoning": 0.000168 + } + } + ] + }, + { + "modelName": "gpt-5.4", + "matchPattern": "(?i)^(openai/)?(gpt-5.4)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 2.5e-7, + "input_cache_read": 2.5e-7, + "output": 0.000015, + "output_reasoning_tokens": 0.000015, + "output_reasoning": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-5.4-pro", + "matchPattern": "(?i)^(openai/)?(gpt-5.4-pro)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00018, + "output_reasoning_tokens": 0.00018, + "output_reasoning": 0.00018 + } + } + ] + }, + { + "modelName": "gpt-5.4-2026-03-05", + "matchPattern": "(?i)^(openai/)?(gpt-5.4-2026-03-05)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.0000025, + "input_cached_tokens": 2.5e-7, + "input_cache_read": 2.5e-7, + "output": 0.000015, + "output_reasoning_tokens": 0.000015, + "output_reasoning": 0.000015 + } + } + ] + }, + { + "modelName": "gpt-5.4-pro-2026-03-05", + "matchPattern": "(?i)^(openai/)?(gpt-5.4-pro-2026-03-05)$", + "startDate": "2026-03-05T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 0.00003, + "output": 0.00018, + "output_reasoning_tokens": 0.00018, + "output_reasoning": 0.00018 + } + } + ] + }, + { + "modelName": "gemini-3-flash-preview", + "matchPattern": "(?i)^(google/)?(gemini-3-flash-preview)$", + "startDate": "2025-12-21T12:01:42.282Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 5e-7, + "input_modality_1": 5e-7, + "prompt_token_count": 5e-7, + "promptTokenCount": 5e-7, + "input_cached_tokens": 5e-8, + "cached_content_token_count": 5e-8, + "output": 0.000003, + "output_modality_1": 0.000003, + "candidates_token_count": 0.000003, + "candidatesTokenCount": 0.000003, + "thoughtsTokenCount": 0.000003, + "thoughts_token_count": 0.000003, + "output_reasoning": 0.000003 + } + } + ] + }, + { + "modelName": "gemini-3.1-flash-lite-preview", + "matchPattern": "(?i)^(google/)?(gemini-3.1-flash-lite-preview)$", + "startDate": "2026-03-03T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input": 2.5e-7, + "input_modality_1": 2.5e-7, + "prompt_token_count": 2.5e-7, + "promptTokenCount": 2.5e-7, + "input_cached_tokens": 2.5e-8, + "cached_content_token_count": 2.5e-8, + "output": 0.0000015, + "output_modality_1": 0.0000015, + "candidates_token_count": 0.0000015, + "candidatesTokenCount": 0.0000015, + "thoughtsTokenCount": 0.0000015, + "thoughts_token_count": 0.0000015, + "output_reasoning": 0.0000015, + "input_audio_tokens": 5e-7 + } + } + ] + } +]; diff --git a/internal-packages/llm-pricing/src/index.ts b/internal-packages/llm-pricing/src/index.ts new file mode 100644 index 00000000000..3632434c137 --- /dev/null +++ b/internal-packages/llm-pricing/src/index.ts @@ -0,0 +1,11 @@ +export { ModelPricingRegistry } from "./registry.js"; +export { seedLlmPricing } from "./seed.js"; +export { defaultModelPrices } from "./defaultPrices.js"; +export type { + LlmModelWithPricing, + LlmCostResult, + LlmPricingTierWithPrices, + LlmPriceEntry, + PricingCondition, + DefaultModelDefinition, +} from "./types.js"; diff --git a/internal-packages/llm-pricing/src/registry.test.ts b/internal-packages/llm-pricing/src/registry.test.ts new file mode 100644 index 00000000000..7732c596493 --- /dev/null +++ b/internal-packages/llm-pricing/src/registry.test.ts @@ -0,0 +1,233 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { ModelPricingRegistry } from "./registry.js"; +import { defaultModelPrices } from "./defaultPrices.js"; +import type { LlmModelWithPricing } from "./types.js"; + +// Convert POSIX-style (?i) inline flag to JS RegExp 'i' flag +function compilePattern(pattern: string): RegExp { + if (pattern.startsWith("(?i)")) { + return new RegExp(pattern.slice(4), "i"); + } + return new RegExp(pattern); +} + +// Create a mock registry that we can load with test data without Prisma +class TestableRegistry extends ModelPricingRegistry { + loadPatterns(models: LlmModelWithPricing[]) { + // Access private fields via any cast for testing + const self = this as any; + self._patterns = models.map((model) => ({ + regex: compilePattern(model.matchPattern), + model, + })); + self._exactMatchCache = new Map(); + self._loaded = true; + } +} + +const gpt4o: LlmModelWithPricing = { + id: "model-gpt4o", + modelName: "gpt-4o", + matchPattern: "^gpt-4o(-\\d{4}-\\d{2}-\\d{2})?$", + startDate: null, + pricingTiers: [ + { + id: "tier-gpt4o-standard", + name: "Standard", + isDefault: true, + priority: 0, + conditions: [], + prices: [ + { usageType: "input", price: 0.0000025 }, + { usageType: "output", price: 0.00001 }, + { usageType: "input_cached_tokens", price: 0.00000125 }, + ], + }, + ], +}; + +const claudeSonnet: LlmModelWithPricing = { + id: "model-claude-sonnet", + modelName: "claude-sonnet-4-0", + matchPattern: "^claude-sonnet-4-0(-\\d{8})?$", + startDate: null, + pricingTiers: [ + { + id: "tier-claude-sonnet-standard", + name: "Standard", + isDefault: true, + priority: 0, + conditions: [], + prices: [ + { usageType: "input", price: 0.000003 }, + { usageType: "output", price: 0.000015 }, + { usageType: "input_cached_tokens", price: 0.0000015 }, + ], + }, + ], +}; + +describe("ModelPricingRegistry", () => { + let registry: TestableRegistry; + + beforeEach(() => { + registry = new TestableRegistry(null as any); + registry.loadPatterns([gpt4o, claudeSonnet]); + }); + + describe("match", () => { + it("should match exact model name", () => { + const result = registry.match("gpt-4o"); + expect(result).not.toBeNull(); + expect(result!.modelName).toBe("gpt-4o"); + }); + + it("should match model with date suffix", () => { + const result = registry.match("gpt-4o-2024-08-06"); + expect(result).not.toBeNull(); + expect(result!.modelName).toBe("gpt-4o"); + }); + + it("should match claude model", () => { + const result = registry.match("claude-sonnet-4-0-20250514"); + expect(result).not.toBeNull(); + expect(result!.modelName).toBe("claude-sonnet-4-0"); + }); + + it("should return null for unknown model", () => { + const result = registry.match("unknown-model-xyz"); + expect(result).toBeNull(); + }); + + it("should cache exact matches", () => { + registry.match("gpt-4o"); + registry.match("gpt-4o"); + // Second call should use cache - no way to verify without mocking, but it shouldn't error + expect(registry.match("gpt-4o")!.modelName).toBe("gpt-4o"); + }); + + it("should cache misses", () => { + expect(registry.match("unknown")).toBeNull(); + expect(registry.match("unknown")).toBeNull(); + }); + }); + + describe("calculateCost", () => { + it("should calculate cost for input and output tokens", () => { + const result = registry.calculateCost("gpt-4o", { + input: 1000, + output: 100, + }); + + expect(result).not.toBeNull(); + expect(result!.matchedModelName).toBe("gpt-4o"); + expect(result!.pricingTierName).toBe("Standard"); + expect(result!.inputCost).toBeCloseTo(0.0025); // 1000 * 0.0000025 + expect(result!.outputCost).toBeCloseTo(0.001); // 100 * 0.00001 + expect(result!.totalCost).toBeCloseTo(0.0035); + }); + + it("should include cached token costs", () => { + const result = registry.calculateCost("gpt-4o", { + input: 500, + output: 50, + input_cached_tokens: 200, + }); + + expect(result).not.toBeNull(); + expect(result!.costDetails["input"]).toBeCloseTo(0.00125); // 500 * 0.0000025 + expect(result!.costDetails["output"]).toBeCloseTo(0.0005); // 50 * 0.00001 + expect(result!.costDetails["input_cached_tokens"]).toBeCloseTo(0.00025); // 200 * 0.00000125 + expect(result!.totalCost).toBeCloseTo(0.002); + }); + + it("should return null for unknown model", () => { + const result = registry.calculateCost("unknown-model", { input: 100, output: 50 }); + expect(result).toBeNull(); + }); + + it("should handle zero tokens", () => { + const result = registry.calculateCost("gpt-4o", { input: 0, output: 0 }); + expect(result).not.toBeNull(); + expect(result!.totalCost).toBe(0); + }); + + it("should handle missing usage types gracefully", () => { + const result = registry.calculateCost("gpt-4o", { input: 100 }); + expect(result).not.toBeNull(); + expect(result!.inputCost).toBeCloseTo(0.00025); + expect(result!.outputCost).toBe(0); // No output tokens + expect(result!.totalCost).toBeCloseTo(0.00025); + }); + }); + + describe("isLoaded", () => { + it("should return false before loading", () => { + const freshRegistry = new TestableRegistry(null as any); + expect(freshRegistry.isLoaded).toBe(false); + }); + + it("should return true after loading", () => { + expect(registry.isLoaded).toBe(true); + }); + }); + + describe("defaultModelPrices (Langfuse JSON)", () => { + it("should load all models from the JSON file", () => { + expect(defaultModelPrices.length).toBeGreaterThan(100); + }); + + it("should compile all match patterns without errors", () => { + const langfuseRegistry = new TestableRegistry(null as any); + const models: LlmModelWithPricing[] = defaultModelPrices.map((def, i) => ({ + id: `test-${i}`, + modelName: def.modelName, + matchPattern: def.matchPattern, + startDate: def.startDate ? new Date(def.startDate) : null, + pricingTiers: def.pricingTiers.map((tier, j) => ({ + id: `tier-${i}-${j}`, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: Object.entries(tier.prices).map(([usageType, price]) => ({ + usageType, + price, + })), + })), + })); + + // This should not throw — all 141 patterns should compile + expect(() => langfuseRegistry.loadPatterns(models)).not.toThrow(); + expect(langfuseRegistry.isLoaded).toBe(true); + }); + + it("should match real-world model names from Langfuse patterns", () => { + const langfuseRegistry = new TestableRegistry(null as any); + const models: LlmModelWithPricing[] = defaultModelPrices.map((def, i) => ({ + id: `test-${i}`, + modelName: def.modelName, + matchPattern: def.matchPattern, + startDate: null, + pricingTiers: def.pricingTiers.map((tier, j) => ({ + id: `tier-${i}-${j}`, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: Object.entries(tier.prices).map(([usageType, price]) => ({ + usageType, + price, + })), + })), + })); + langfuseRegistry.loadPatterns(models); + + // Test real model strings that SDKs send + expect(langfuseRegistry.match("gpt-4o")).not.toBeNull(); + expect(langfuseRegistry.match("gpt-4o-mini")).not.toBeNull(); + expect(langfuseRegistry.match("claude-sonnet-4-5-20250929")).not.toBeNull(); + expect(langfuseRegistry.match("claude-sonnet-4-20250514")).not.toBeNull(); + }); + }); +}); diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts new file mode 100644 index 00000000000..85713e5268a --- /dev/null +++ b/internal-packages/llm-pricing/src/registry.ts @@ -0,0 +1,209 @@ +import type { PrismaClient } from "@trigger.dev/database"; +import { trail } from "agentcrumbs"; // @crumbs +import type { + LlmModelWithPricing, + LlmCostResult, + LlmPricingTierWithPrices, + PricingCondition, +} from "./types.js"; + +const crumb = trail("webapp:llm-pricing"); // @crumbs + +type CompiledPattern = { + regex: RegExp; + model: LlmModelWithPricing; +}; + +// Convert POSIX-style (?i) inline flag to JS RegExp 'i' flag +function compilePattern(pattern: string): RegExp { + if (pattern.startsWith("(?i)")) { + return new RegExp(pattern.slice(4), "i"); + } + return new RegExp(pattern); +} + +export class ModelPricingRegistry { + private _prisma: PrismaClient; + private _patterns: CompiledPattern[] = []; + private _exactMatchCache: Map = new Map(); + private _loaded = false; + + constructor(prisma: PrismaClient) { + this._prisma = prisma; + } + + get isLoaded(): boolean { + return this._loaded; + } + + async loadFromDatabase(): Promise { + const models = await this._prisma.llmModel.findMany({ + where: { projectId: null }, + include: { + pricingTiers: { + include: { prices: true }, + orderBy: { priority: "asc" }, + }, + }, + orderBy: [{ startDate: "desc" }], + }); + + crumb("loaded models from db", { count: models.length }); // @crumbs + + const compiled: CompiledPattern[] = []; + let skippedCount = 0; // @crumbs + + for (const model of models) { + try { + const regex = compilePattern(model.matchPattern); + const tiers: LlmPricingTierWithPrices[] = model.pricingTiers.map((tier) => ({ + id: tier.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: (tier.conditions as PricingCondition[]) ?? [], + prices: tier.prices.map((p) => ({ + usageType: p.usageType, + price: Number(p.price), + })), + })); + + compiled.push({ + regex, + model: { + id: model.id, + modelName: model.modelName, + matchPattern: model.matchPattern, + startDate: model.startDate, + pricingTiers: tiers, + }, + }); + } catch { + skippedCount++; // @crumbs + // Skip models with invalid regex patterns + console.warn(`Invalid regex pattern for model ${model.modelName}: ${model.matchPattern}`); + crumb("invalid regex pattern", { modelName: model.modelName, pattern: model.matchPattern }); // @crumbs + } + } + + this._patterns = compiled; + this._exactMatchCache.clear(); + this._loaded = true; + crumb("registry loaded", { patterns: compiled.length, skipped: skippedCount }); // @crumbs + } + + async reload(): Promise { + await this.loadFromDatabase(); + } + + match(responseModel: string): LlmModelWithPricing | null { + if (!this._loaded) return null; + + // Check exact match cache + const cached = this._exactMatchCache.get(responseModel); + if (cached !== undefined) return cached; + + // Iterate compiled regex patterns + for (const { regex, model } of this._patterns) { + if (regex.test(responseModel)) { + this._exactMatchCache.set(responseModel, model); + crumb("model matched", { responseModel, matchedModel: model.modelName }); // @crumbs + return model; + } + } + + // Cache miss + this._exactMatchCache.set(responseModel, null); + crumb("model not matched", { responseModel }); // @crumbs + return null; + } + + calculateCost( + responseModel: string, + usageDetails: Record + ): LlmCostResult | null { + const model = this.match(responseModel); + if (!model) return null; + + const tier = this._matchPricingTier(model.pricingTiers, usageDetails); + if (!tier) return null; + + const costDetails: Record = {}; + let totalCost = 0; + + for (const priceEntry of tier.prices) { + const tokenCount = usageDetails[priceEntry.usageType] ?? 0; + if (tokenCount === 0) continue; + const cost = tokenCount * priceEntry.price; + costDetails[priceEntry.usageType] = cost; + totalCost += cost; + } + + const inputCost = costDetails["input"] ?? 0; + const outputCost = costDetails["output"] ?? 0; + + return { + matchedModelId: model.id, + matchedModelName: model.modelName, + pricingTierId: tier.id, + pricingTierName: tier.name, + inputCost, + outputCost, + totalCost, + costDetails, + }; + } + + private _matchPricingTier( + tiers: LlmPricingTierWithPrices[], + usageDetails: Record + ): LlmPricingTierWithPrices | null { + if (tiers.length === 0) return null; + + // Tiers are sorted by priority ascending (lowest first) + // Evaluate conditions — first tier whose conditions match wins + for (const tier of tiers) { + if (tier.conditions.length === 0) { + // No conditions = default tier + return tier; + } + + if (this._evaluateConditions(tier.conditions, usageDetails)) { + return tier; + } + } + + // Fallback to default tier + const defaultTier = tiers.find((t) => t.isDefault); + return defaultTier ?? tiers[0] ?? null; + } + + private _evaluateConditions( + conditions: PricingCondition[], + usageDetails: Record + ): boolean { + return conditions.every((condition) => { + // Find matching usage detail key + const regex = new RegExp(condition.usageDetailPattern); + const matchingValue = Object.entries(usageDetails).find(([key]) => regex.test(key)); + const value = matchingValue?.[1] ?? 0; + + switch (condition.operator) { + case "gt": + return value > condition.value; + case "gte": + return value >= condition.value; + case "lt": + return value < condition.value; + case "lte": + return value <= condition.value; + case "eq": + return value === condition.value; + case "neq": + return value !== condition.value; + default: + return false; + } + }); + } +} diff --git a/internal-packages/llm-pricing/src/seed.ts b/internal-packages/llm-pricing/src/seed.ts new file mode 100644 index 00000000000..6289d31176b --- /dev/null +++ b/internal-packages/llm-pricing/src/seed.ts @@ -0,0 +1,59 @@ +import type { PrismaClient } from "@trigger.dev/database"; +import { defaultModelPrices } from "./defaultPrices.js"; + +export async function seedLlmPricing(prisma: PrismaClient): Promise<{ + modelsCreated: number; + modelsSkipped: number; +}> { + let modelsCreated = 0; + let modelsSkipped = 0; + + for (const modelDef of defaultModelPrices) { + // Check if this model already exists (don't overwrite admin changes) + const existing = await prisma.llmModel.findFirst({ + where: { + projectId: null, + modelName: modelDef.modelName, + }, + }); + + if (existing) { + modelsSkipped++; + continue; + } + + // Create model first + const model = await prisma.llmModel.create({ + data: { + modelName: modelDef.modelName, + matchPattern: modelDef.matchPattern, + startDate: modelDef.startDate ? new Date(modelDef.startDate) : null, + source: "default", + }, + }); + + // Create tiers and prices with explicit model connection + for (const tier of modelDef.pricingTiers) { + await prisma.llmPricingTier.create({ + data: { + modelId: model.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId: model.id, + usageType, + price, + })), + }, + }, + }); + } + + modelsCreated++; + } + + return { modelsCreated, modelsSkipped }; +} diff --git a/internal-packages/llm-pricing/src/types.ts b/internal-packages/llm-pricing/src/types.ts new file mode 100644 index 00000000000..5e2ffdcaec5 --- /dev/null +++ b/internal-packages/llm-pricing/src/types.ts @@ -0,0 +1,53 @@ +import type { Decimal } from "@trigger.dev/database"; + +export type PricingCondition = { + usageDetailPattern: string; + operator: "gt" | "gte" | "lt" | "lte" | "eq" | "neq"; + value: number; +}; + +export type LlmPriceEntry = { + usageType: string; + price: number; +}; + +export type LlmPricingTierWithPrices = { + id: string; + name: string; + isDefault: boolean; + priority: number; + conditions: PricingCondition[]; + prices: LlmPriceEntry[]; +}; + +export type LlmModelWithPricing = { + id: string; + modelName: string; + matchPattern: string; + startDate: Date | null; + pricingTiers: LlmPricingTierWithPrices[]; +}; + +export type LlmCostResult = { + matchedModelId: string; + matchedModelName: string; + pricingTierId: string; + pricingTierName: string; + inputCost: number; + outputCost: number; + totalCost: number; + costDetails: Record; +}; + +export type DefaultModelDefinition = { + modelName: string; + matchPattern: string; + startDate?: string; + pricingTiers: Array<{ + name: string; + isDefault: boolean; + priority: number; + conditions: PricingCondition[]; + prices: Record; + }>; +}; diff --git a/internal-packages/llm-pricing/tsconfig.json b/internal-packages/llm-pricing/tsconfig.json new file mode 100644 index 00000000000..c64cf33133b --- /dev/null +++ b/internal-packages/llm-pricing/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2019", + "lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"], + "module": "Node16", + "moduleResolution": "Node16", + "moduleDetection": "force", + "verbatimModuleSyntax": false, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "preserveWatchOutput": true, + "skipLibCheck": true, + "noEmit": true, + "strict": true, + "resolveJsonModule": true + }, + "exclude": ["node_modules"] +} diff --git a/packages/core/src/v3/schemas/style.ts b/packages/core/src/v3/schemas/style.ts index eab62c5b41b..2f833b800ac 100644 --- a/packages/core/src/v3/schemas/style.ts +++ b/packages/core/src/v3/schemas/style.ts @@ -11,11 +11,12 @@ const AccessoryItem = z.object({ text: z.string(), variant: z.string().optional(), url: z.string().optional(), + icon: z.string().optional(), }); const Accessory = z.object({ items: z.array(AccessoryItem), - style: z.enum(["codepath"]).optional(), + style: z.enum(["codepath", "pills"]).optional(), }); export type Accessory = z.infer; From 3f7722214b5233b3663ad104033e52c9dbe49d7c Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 11 Mar 2026 11:04:21 +0000 Subject: [PATCH 02/30] feat: add friendlyId to LlmModel, TRQL llm_usage integration, and seed-on-startup - Add friendly_id column to llm_models (llm_model_xxx format) - Use friendlyId as matchedModelId in all external surfaces - Add durationNs render type to TSQLResultsTable and QueryResultsChart - Add 4 example queries for llm_usage in query editor - Add LLM_PRICING_SEED_ON_STARTUP env var for local bootstrapping - Update admin API and seed to generate friendlyId refs TRI-7773 --- .../app/components/code/QueryResultsChart.tsx | 5 ++ .../app/components/code/TSQLResultsTable.tsx | 20 +++++++ apps/webapp/app/env.server.ts | 1 + .../ExamplesContent.tsx | 57 +++++++++++++++++++ .../app/routes/admin.api.v1.llm-models.ts | 2 + .../app/v3/llmPricingRegistry.server.ts | 28 +++++---- apps/webapp/test/otlpExporter.test.ts | 2 +- .../migration.sql | 4 ++ .../database/prisma/schema.prisma | 1 + internal-packages/llm-pricing/package.json | 1 + .../llm-pricing/src/registry.test.ts | 4 ++ internal-packages/llm-pricing/src/registry.ts | 3 +- internal-packages/llm-pricing/src/seed.ts | 2 + internal-packages/llm-pricing/src/types.ts | 1 + pnpm-lock.yaml | 12 ++++ 15 files changed, 130 insertions(+), 13 deletions(-) diff --git a/apps/webapp/app/components/code/QueryResultsChart.tsx b/apps/webapp/app/components/code/QueryResultsChart.tsx index 190304a1e9c..aa6bc6d8898 100644 --- a/apps/webapp/app/components/code/QueryResultsChart.tsx +++ b/apps/webapp/app/components/code/QueryResultsChart.tsx @@ -1209,6 +1209,11 @@ function createYAxisFormatter( formatDurationMilliseconds(value * 1000, { style: "short" }); } + if (format === "durationNs") { + return (value: number): string => + formatDurationMilliseconds(value / 1_000_000, { style: "short" }); + } + if (format === "costInDollars" || format === "cost") { return (value: number): string => { const dollars = format === "cost" ? value / 100 : value; diff --git a/apps/webapp/app/components/code/TSQLResultsTable.tsx b/apps/webapp/app/components/code/TSQLResultsTable.tsx index 3eb033c1d09..b2caf74dac6 100644 --- a/apps/webapp/app/components/code/TSQLResultsTable.tsx +++ b/apps/webapp/app/components/code/TSQLResultsTable.tsx @@ -81,6 +81,11 @@ function getFormattedValue(value: unknown, column: OutputColumnMetadata): string return formatDurationMilliseconds(value * 1000, { style: "short" }); } break; + case "durationNs": + if (typeof value === "number") { + return formatDurationMilliseconds(value / 1_000_000, { style: "short" }); + } + break; case "cost": if (typeof value === "number") { return formatCurrencyAccurate(value / 100); @@ -282,6 +287,12 @@ function getDisplayLength(value: unknown, column: OutputColumnMetadata): number return formatted.length; } return 10; + case "durationNs": + if (typeof value === "number") { + const formatted = formatDurationMilliseconds(value / 1_000_000, { style: "short" }); + return formatted.length; + } + return 10; case "cost": case "costInDollars": // Currency format: "$1,234.56" @@ -598,6 +609,15 @@ function CellValue({ ); } return {String(value)}; + case "durationNs": + if (typeof value === "number") { + return ( + + {formatDurationMilliseconds(value / 1_000_000, { style: "short" })} + + ); + } + return {String(value)}; case "cost": if (typeof value === "number") { return {formatCurrencyAccurate(value / 100)}; diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 8877c982ae0..711c37157e5 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1280,6 +1280,7 @@ const EnvironmentSchema = z // LLM cost tracking LLM_COST_TRACKING_ENABLED: BoolEnv.default(true), LLM_PRICING_RELOAD_INTERVAL_MS: z.coerce.number().int().default(5 * 60 * 1000), // 5 minutes + LLM_PRICING_SEED_ON_STARTUP: BoolEnv.default(false), // Bootstrap TRIGGER_BOOTSTRAP_ENABLED: z.string().default("0"), diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx index 4d027223f14..23bac4b97ad 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx @@ -118,6 +118,63 @@ LIMIT 100`, scope: "environment", table: "metrics", }, + { + title: "LLM cost by model (past 7d)", + description: "Total cost, input tokens, and output tokens grouped by model over the last 7 days.", + query: `SELECT + response_model, + SUM(total_cost) AS total_cost, + SUM(input_tokens) AS input_tokens, + SUM(output_tokens) AS output_tokens +FROM llm_usage +WHERE start_time > now() - INTERVAL 7 DAY +GROUP BY response_model +ORDER BY total_cost DESC`, + scope: "environment", + table: "llm_usage", + }, + { + title: "LLM cost over time", + description: "Total LLM cost bucketed over time. The bucket size adjusts automatically.", + query: `SELECT + timeBucket(), + SUM(total_cost) AS total_cost +FROM llm_usage +GROUP BY timeBucket +ORDER BY timeBucket +LIMIT 1000`, + scope: "environment", + table: "llm_usage", + }, + { + title: "Most expensive runs by LLM cost (top 50)", + description: "Top 50 runs by total LLM cost with token breakdown.", + query: `SELECT + run_id, + task_identifier, + SUM(total_cost) AS llm_cost, + SUM(input_tokens) AS input_tokens, + SUM(output_tokens) AS output_tokens +FROM llm_usage +GROUP BY run_id, task_identifier +ORDER BY llm_cost DESC +LIMIT 50`, + scope: "environment", + table: "llm_usage", + }, + { + title: "LLM calls by provider", + description: "Count and cost of LLM calls grouped by AI provider.", + query: `SELECT + gen_ai_system, + count() AS call_count, + SUM(total_cost) AS total_cost +FROM llm_usage +GROUP BY gen_ai_system +ORDER BY total_cost DESC`, + scope: "environment", + table: "llm_usage", + }, ]; const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name })); diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.ts index 7b1b2226ca5..706c73549de 100644 --- a/apps/webapp/app/routes/admin.api.v1.llm-models.ts +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.ts @@ -2,6 +2,7 @@ import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-r import { z } from "zod"; import { prisma } from "~/db.server"; import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { generateFriendlyId } from "~/v3/friendlyIdentifiers"; async function requireAdmin(request: Request) { const authResult = await authenticateApiRequestWithPersonalAccessToken(request); @@ -93,6 +94,7 @@ export async function action({ request }: ActionFunctionArgs) { // Create model first, then tiers with explicit model connection const model = await prisma.llmModel.create({ data: { + friendlyId: generateFriendlyId("llm_model"), modelName, matchPattern, startDate: startDate ? new Date(startDate) : null, diff --git a/apps/webapp/app/v3/llmPricingRegistry.server.ts b/apps/webapp/app/v3/llmPricingRegistry.server.ts index 4a2c67e247b..d05b1afc2c8 100644 --- a/apps/webapp/app/v3/llmPricingRegistry.server.ts +++ b/apps/webapp/app/v3/llmPricingRegistry.server.ts @@ -1,12 +1,23 @@ -import { ModelPricingRegistry } from "@internal/llm-pricing"; +import { ModelPricingRegistry, seedLlmPricing } from "@internal/llm-pricing"; import { trail } from "agentcrumbs"; // @crumbs -import { $replica } from "~/db.server"; +import { prisma, $replica } from "~/db.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; import { setLlmPricingRegistry } from "./utils/enrichCreatableEvents.server"; const crumb = trail("webapp:llm-registry"); // @crumbs +async function initRegistry(registry: ModelPricingRegistry) { + if (env.LLM_PRICING_SEED_ON_STARTUP) { + crumb("seeding llm pricing on startup"); // @crumbs + const result = await seedLlmPricing(prisma); + crumb("seed complete", { modelsCreated: result.modelsCreated, modelsSkipped: result.modelsSkipped }); // @crumbs + } + + await registry.loadFromDatabase(); + crumb("registry loaded successfully", { isLoaded: registry.isLoaded }); // @crumbs +} + export const llmPricingRegistry = singleton("llmPricingRegistry", () => { if (!env.LLM_COST_TRACKING_ENABLED) { crumb("llm cost tracking disabled via env"); // @crumbs @@ -19,15 +30,10 @@ export const llmPricingRegistry = singleton("llmPricingRegistry", () => { // Wire up the registry so enrichCreatableEvents can use it setLlmPricingRegistry(registry); - registry - .loadFromDatabase() - .then(() => { - crumb("registry loaded successfully", { isLoaded: registry.isLoaded }); // @crumbs - }) - .catch((err) => { - crumb("registry load failed", { error: String(err) }); // @crumbs - console.error("Failed to load LLM pricing registry", err); - }); + initRegistry(registry).catch((err) => { + crumb("registry init failed", { error: String(err) }); // @crumbs + console.error("Failed to initialize LLM pricing registry", err); + }); // Periodic reload const reloadInterval = env.LLM_PRICING_RELOAD_INTERVAL_MS; diff --git a/apps/webapp/test/otlpExporter.test.ts b/apps/webapp/test/otlpExporter.test.ts index a1d1baa3b1c..2a569b74d62 100644 --- a/apps/webapp/test/otlpExporter.test.ts +++ b/apps/webapp/test/otlpExporter.test.ts @@ -406,7 +406,7 @@ describe("OTLPExporter", () => { const inputCost = (usageDetails["input"] ?? 0) * 0.0000025; const outputCost = (usageDetails["output"] ?? 0) * 0.00001; return { - matchedModelId: "model-gpt4o", + matchedModelId: "llm_model_gpt4o", matchedModelName: "gpt-4o", pricingTierId: "tier-standard", pricingTierName: "Standard", diff --git a/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql b/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql index 44cc581616f..286de6eacfb 100644 --- a/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql +++ b/internal-packages/database/prisma/migrations/20260310155049_add_llm_pricing_tables/migration.sql @@ -1,6 +1,7 @@ -- CreateTable CREATE TABLE "public"."llm_models" ( "id" TEXT NOT NULL, + "friendly_id" TEXT NOT NULL, "project_id" TEXT, "model_name" TEXT NOT NULL, "match_pattern" TEXT NOT NULL, @@ -35,6 +36,9 @@ CREATE TABLE "public"."llm_prices" ( CONSTRAINT "llm_prices_pkey" PRIMARY KEY ("id") ); +-- CreateIndex +CREATE UNIQUE INDEX "llm_models_friendly_id_key" ON "public"."llm_models"("friendly_id"); + -- CreateIndex CREATE INDEX "llm_models_project_id_idx" ON "public"."llm_models"("project_id"); diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 65878a8af7c..9e91fc70f14 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2586,6 +2586,7 @@ model MetricsDashboard { /// A known LLM model or model pattern for cost tracking model LlmModel { id String @id @default(cuid()) + friendlyId String @unique @map("friendly_id") projectId String? @map("project_id") project Project? @relation(fields: [projectId], references: [id], onDelete: Cascade) modelName String @map("model_name") diff --git a/internal-packages/llm-pricing/package.json b/internal-packages/llm-pricing/package.json index 423dae77eb4..2ac51427dd6 100644 --- a/internal-packages/llm-pricing/package.json +++ b/internal-packages/llm-pricing/package.json @@ -6,6 +6,7 @@ "types": "./src/index.ts", "type": "module", "dependencies": { + "@trigger.dev/core": "workspace:*", "@trigger.dev/database": "workspace:*" }, "scripts": { diff --git a/internal-packages/llm-pricing/src/registry.test.ts b/internal-packages/llm-pricing/src/registry.test.ts index 7732c596493..93d6a362cb3 100644 --- a/internal-packages/llm-pricing/src/registry.test.ts +++ b/internal-packages/llm-pricing/src/registry.test.ts @@ -27,6 +27,7 @@ class TestableRegistry extends ModelPricingRegistry { const gpt4o: LlmModelWithPricing = { id: "model-gpt4o", + friendlyId: "llm_model_gpt4o", modelName: "gpt-4o", matchPattern: "^gpt-4o(-\\d{4}-\\d{2}-\\d{2})?$", startDate: null, @@ -48,6 +49,7 @@ const gpt4o: LlmModelWithPricing = { const claudeSonnet: LlmModelWithPricing = { id: "model-claude-sonnet", + friendlyId: "llm_model_claude_sonnet", modelName: "claude-sonnet-4-0", matchPattern: "^claude-sonnet-4-0(-\\d{8})?$", startDate: null, @@ -181,6 +183,7 @@ describe("ModelPricingRegistry", () => { const langfuseRegistry = new TestableRegistry(null as any); const models: LlmModelWithPricing[] = defaultModelPrices.map((def, i) => ({ id: `test-${i}`, + friendlyId: `llm_model_test${i}`, modelName: def.modelName, matchPattern: def.matchPattern, startDate: def.startDate ? new Date(def.startDate) : null, @@ -206,6 +209,7 @@ describe("ModelPricingRegistry", () => { const langfuseRegistry = new TestableRegistry(null as any); const models: LlmModelWithPricing[] = defaultModelPrices.map((def, i) => ({ id: `test-${i}`, + friendlyId: `llm_model_test${i}`, modelName: def.modelName, matchPattern: def.matchPattern, startDate: null, diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts index 85713e5268a..506a9360bd2 100644 --- a/internal-packages/llm-pricing/src/registry.ts +++ b/internal-packages/llm-pricing/src/registry.ts @@ -72,6 +72,7 @@ export class ModelPricingRegistry { regex, model: { id: model.id, + friendlyId: model.friendlyId, modelName: model.modelName, matchPattern: model.matchPattern, startDate: model.startDate, @@ -143,7 +144,7 @@ export class ModelPricingRegistry { const outputCost = costDetails["output"] ?? 0; return { - matchedModelId: model.id, + matchedModelId: model.friendlyId, matchedModelName: model.modelName, pricingTierId: tier.id, pricingTierName: tier.name, diff --git a/internal-packages/llm-pricing/src/seed.ts b/internal-packages/llm-pricing/src/seed.ts index 6289d31176b..b4f95373eff 100644 --- a/internal-packages/llm-pricing/src/seed.ts +++ b/internal-packages/llm-pricing/src/seed.ts @@ -1,4 +1,5 @@ import type { PrismaClient } from "@trigger.dev/database"; +import { generateFriendlyId } from "@trigger.dev/core/v3/isomorphic"; import { defaultModelPrices } from "./defaultPrices.js"; export async function seedLlmPricing(prisma: PrismaClient): Promise<{ @@ -25,6 +26,7 @@ export async function seedLlmPricing(prisma: PrismaClient): Promise<{ // Create model first const model = await prisma.llmModel.create({ data: { + friendlyId: generateFriendlyId("llm_model"), modelName: modelDef.modelName, matchPattern: modelDef.matchPattern, startDate: modelDef.startDate ? new Date(modelDef.startDate) : null, diff --git a/internal-packages/llm-pricing/src/types.ts b/internal-packages/llm-pricing/src/types.ts index 5e2ffdcaec5..2deec6246ed 100644 --- a/internal-packages/llm-pricing/src/types.ts +++ b/internal-packages/llm-pricing/src/types.ts @@ -22,6 +22,7 @@ export type LlmPricingTierWithPrices = { export type LlmModelWithPricing = { id: string; + friendlyId: string; modelName: string; matchPattern: string; startDate: Date | null; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 88ac6ad5421..854d4215447 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -302,6 +302,9 @@ importers: '@internal/cache': specifier: workspace:* version: link:../../internal-packages/cache + '@internal/llm-pricing': + specifier: workspace:* + version: link:../../internal-packages/llm-pricing '@internal/redis': specifier: workspace:* version: link:../../internal-packages/redis @@ -1125,6 +1128,15 @@ importers: specifier: 18.2.69 version: 18.2.69 + internal-packages/llm-pricing: + dependencies: + '@trigger.dev/core': + specifier: workspace:* + version: link:../../packages/core + '@trigger.dev/database': + specifier: workspace:* + version: link:../database + internal-packages/otlp-importer: dependencies: long: From 9d5c204aed0f5d19025464d707769a9e20131e6c Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 11 Mar 2026 13:13:04 +0000 Subject: [PATCH 03/30] feat: add metadata map to llm_usage_v1 for cost attribution by user/tenant --- .changeset/llm-metadata-run-tags.md | 5 ++ .../ExamplesContent.tsx | 33 ++++++++++++ .../clickhouseEventRepository.server.ts | 3 +- .../eventRepository/eventRepository.types.ts | 2 + apps/webapp/app/v3/otlpExporter.server.ts | 21 ++++++++ apps/webapp/app/v3/querySchemas.ts | 9 ++++ .../v3/utils/enrichCreatableEvents.server.ts | 53 ++++++++++++++++++- .../schema/024_create_llm_usage_v1.sql | 5 +- internal-packages/clickhouse/src/llmUsage.ts | 2 + internal-packages/tsql/src/query/printer.ts | 15 ++++++ .../core/src/v3/taskContext/otelProcessors.ts | 6 +++ 11 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 .changeset/llm-metadata-run-tags.md diff --git a/.changeset/llm-metadata-run-tags.md b/.changeset/llm-metadata-run-tags.md new file mode 100644 index 00000000000..85f04c363b8 --- /dev/null +++ b/.changeset/llm-metadata-run-tags.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Propagate run tags to span attributes so they can be extracted server-side for LLM cost attribution metadata. diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx index 23bac4b97ad..1235b443348 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx @@ -175,6 +175,39 @@ ORDER BY total_cost DESC`, scope: "environment", table: "llm_usage", }, + { + title: "LLM cost by user", + description: + "Total LLM cost per user from run tags or AI SDK telemetry metadata. Uses metadata.userId which comes from experimental_telemetry metadata or run tags like user:123.", + query: `SELECT + metadata.userId AS user_id, + SUM(total_cost) AS total_cost, + SUM(total_tokens) AS total_tokens, + count() AS call_count +FROM llm_usage +WHERE metadata.userId != '' +GROUP BY metadata.userId +ORDER BY total_cost DESC +LIMIT 50`, + scope: "environment", + table: "llm_usage", + }, + { + title: "LLM cost by metadata key", + description: + "Browse all metadata keys and their LLM cost. Metadata comes from run tags (key:value) and AI SDK telemetry metadata.", + query: `SELECT + metadata, + response_model, + total_cost, + total_tokens, + run_id +FROM llm_usage +ORDER BY start_time DESC +LIMIT 20`, + scope: "environment", + table: "llm_usage", + }, ]; const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name })); diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts index 780b74d85e1..7a46b6b3bfb 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts @@ -279,6 +279,7 @@ export class ClickhouseEventRepository implements IEventRepository { output_cost: llmUsage.outputCost, total_cost: llmUsage.totalCost, cost_details: llmUsage.costDetails, + metadata: llmUsage.metadata, start_time: this.#clampAndFormatStartTime(event.startTime.toString()), duration: formatClickhouseUnsignedIntegerString(event.duration ?? 0), }; @@ -311,7 +312,7 @@ export class ClickhouseEventRepository implements IEventRepository { .map((e) => this.#createLlmUsageInput(e)); if (llmUsageRows.length > 0) { - crumb("queuing llm usage rows", { count: llmUsageRows.length, firstRunId: llmUsageRows[0]?.run_id }); // @crumbs + crumb("queuing llm usage rows", { count: llmUsageRows.length, firstRunId: llmUsageRows[0]?.run_id, metadataKeys: llmUsageRows.map((r) => Object.keys(r.metadata)) }); // @crumbs this._llmUsageFlushScheduler.addToBatch(llmUsageRows); } } diff --git a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts index b750786a651..69590dc9493 100644 --- a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts +++ b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts @@ -37,6 +37,7 @@ export type LlmUsageData = { outputCost: number; totalCost: number; costDetails: Record; + metadata: Record; }; export type CreateEventInput = Omit< @@ -75,6 +76,7 @@ export type CreateEventInput = Omit< metadata: Attributes | undefined; style: Attributes | undefined; machineId?: string; + runTags?: string[]; /** Side-channel data for LLM cost tracking, populated by enrichCreatableEvents */ _llmUsage?: LlmUsageData; }; diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index 0079cb5f0be..b5b5fee70fc 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -38,6 +38,8 @@ import type { import { startSpan } from "./tracing.server"; import { enrichCreatableEvents } from "./utils/enrichCreatableEvents.server"; import "./llmPricingRegistry.server"; // Initialize LLM pricing registry on startup +import { trail } from "agentcrumbs"; // @crumbs +const crumbOtlp = trail("webapp:otlp-exporter"); // @crumbs import { env } from "~/env.server"; import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings"; import { singleton } from "~/utils/singleton"; @@ -392,6 +394,9 @@ function convertSpansToCreateableEvents( SemanticInternalAttributes.METADATA ); + const runTags = extractArrayAttribute(span.attributes ?? [], SemanticInternalAttributes.RUN_TAGS); + if (runTags && runTags.length > 0) { crumbOtlp("extracted runTags from span", { runTags, spanId: binaryToHex(span.spanId) }); } // @crumbs + const properties = truncateAttributes( convertKeyValueItemsToMap(span.attributes ?? [], [], undefined, [ @@ -440,6 +445,7 @@ function convertSpansToCreateableEvents( runId: spanProperties.runId ?? resourceProperties.runId ?? "unknown", taskSlug: spanProperties.taskSlug ?? resourceProperties.taskSlug ?? "unknown", machineId: spanProperties.machineId ?? resourceProperties.machineId, + runTags, attemptNumber: extractNumberAttribute( span.attributes ?? [], @@ -1001,6 +1007,21 @@ function extractBooleanAttribute( return isBoolValue(attribute?.value) ? attribute.value.boolValue : fallback; } +function extractArrayAttribute( + attributes: KeyValue[], + name: string | Array +): string[] | undefined { + const key = Array.isArray(name) ? name.filter(Boolean).join(".") : name; + + const attribute = attributes.find((attribute) => attribute.key === key); + + if (!attribute?.value?.arrayValue?.values) return undefined; + + return attribute.value.arrayValue.values + .filter((v): v is { stringValue: string } => isStringValue(v)) + .map((v) => v.stringValue); +} + function isPartialSpan(span: Span): boolean { if (!span.attributes) return false; diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 323310ebf84..c719ac21378 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -740,6 +740,15 @@ export const llmUsageSchema: TableSchema = { customRenderType: "durationNs", }), }, + metadata: { + name: "metadata", + ...column("Map(LowCardinality(String), String)", { + description: + "Key-value metadata from run tags (key:value format) and AI SDK telemetry metadata. Access keys with dot notation (metadata.userId) or bracket syntax (metadata['userId']).", + example: "{'userId':'user_123','org':'acme'}", + coreColumn: true, + }), + }, }, }; diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index 9dfe091aa10..d76fdda5879 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -48,7 +48,24 @@ function enrichLlmCost(event: CreateEventInput): void { const props = event.properties; if (!props) return; - crumb("enrichLlmCost called", { kind: event.kind, isPartial: event.isPartial, spanId: event.spanId, message: event.message, props }); // @crumbs + // #region @crumbs + // Log all spans (not just gen_ai) that have conversation/chat/session/user context + if (!event.isPartial) { + const contextKeys = Object.entries(props).filter(([k]) => + k.startsWith("ai.telemetry.") || k.startsWith("gen_ai.conversation") || + k.startsWith("chat.") || k.includes("session") || k.includes("user") + ); + if (contextKeys.length > 0) { + crumb("span with context", { + spanId: event.spanId, + parentId: event.parentId, + runId: event.runId, + message: event.message, + contextAttrs: Object.fromEntries(contextKeys), + }); + } + } + // #endregion @crumbs // Only enrich span-like events (INTERNAL, SERVER, CLIENT, CONSUMER, PRODUCER — not LOG, UNSPECIFIED) const enrichableKinds = new Set(["INTERNAL", "SERVER", "CLIENT", "CONSUMER", "PRODUCER"]); @@ -130,6 +147,39 @@ function enrichLlmCost(event: CreateEventInput): void { }, }; + // Build metadata map from run tags and ai.telemetry.metadata.* + const metadata: Record = {}; + + if (event.runTags) { + for (const tag of event.runTags) { + const colonIdx = tag.indexOf(":"); + if (colonIdx > 0) { + metadata[tag.substring(0, colonIdx)] = tag.substring(colonIdx + 1); + } + } + } + + for (const [key, value] of Object.entries(props)) { + if (key.startsWith("ai.telemetry.metadata.") && typeof value === "string") { + metadata[key.slice("ai.telemetry.metadata.".length)] = value; + } + } + + // #region @crumbs + const metadataKeyCount = Object.keys(metadata).length; + if (metadataKeyCount > 0) { + crumb("llm metadata built", { + spanId: event.spanId, + runId: event.runId, + responseModel, + metadataKeyCount, + metadataKeys: Object.keys(metadata), + fromRunTags: event.runTags?.length ?? 0, + fromTelemetry: metadataKeyCount - (event.runTags?.filter((t) => t.includes(":")).length ?? 0), + }); + } + // #endregion @crumbs + // Set _llmUsage side-channel for dual-write to llm_usage_v1 const llmUsage: LlmUsageData = { genAiSystem: (props["gen_ai.system"] as string) ?? "unknown", @@ -147,6 +197,7 @@ function enrichLlmCost(event: CreateEventInput): void { outputCost: cost.outputCost, totalCost: cost.totalCost, costDetails: cost.costDetails, + metadata, }; event._llmUsage = llmUsage; diff --git a/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql index 65bd9c68ebd..5d7f879fdd3 100644 --- a/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql +++ b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql @@ -27,13 +27,16 @@ CREATE TABLE IF NOT EXISTS trigger_dev.llm_usage_v1 total_cost Decimal64(12) DEFAULT 0, cost_details Map(LowCardinality(String), Decimal64(12)), + metadata Map(LowCardinality(String), String), + start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)), duration UInt64 DEFAULT 0 CODEC(ZSTD(1)), inserted_at DateTime64(3) DEFAULT now64(3), INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1 + INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_metadata_keys mapKeys(metadata) TYPE bloom_filter(0.01) GRANULARITY 1 ) ENGINE = MergeTree PARTITION BY toDate(inserted_at) diff --git a/internal-packages/clickhouse/src/llmUsage.ts b/internal-packages/clickhouse/src/llmUsage.ts index fd9deccb562..e9423962ac4 100644 --- a/internal-packages/clickhouse/src/llmUsage.ts +++ b/internal-packages/clickhouse/src/llmUsage.ts @@ -28,6 +28,8 @@ export const LlmUsageV1Input = z.object({ total_cost: z.number(), cost_details: z.record(z.string(), z.number()), + metadata: z.record(z.string(), z.string()), + start_time: z.string(), duration: z.string(), }); diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts index b6e9547db06..d45002f6715 100644 --- a/internal-packages/tsql/src/query/printer.ts +++ b/internal-packages/tsql/src/query/printer.ts @@ -2370,6 +2370,21 @@ export class ClickHousePrinter { // Try to resolve column names through table context const resolvedChain = this.resolveFieldChain(chainWithPrefix); + // For Map columns, convert dot-notation to bracket syntax: + // metadata.user -> metadata['user'] + if (resolvedChain.length > 1) { + const rootColumnSchema = this.resolveFieldToColumnSchema([node.chain[0]]); + if (rootColumnSchema?.type.startsWith("Map(")) { + const rootCol = this.printIdentifierOrIndex(resolvedChain[0]); + const mapKeys = resolvedChain.slice(1); + let result = rootCol; + for (const key of mapKeys) { + result = `${result}[${this.context.addValue(String(key))}]`; + } + return result; + } + } + // Print each chain element let result = resolvedChain.map((part) => this.printIdentifierOrIndex(part)).join("."); diff --git a/packages/core/src/v3/taskContext/otelProcessors.ts b/packages/core/src/v3/taskContext/otelProcessors.ts index 096f7c0ce76..1c0958d655d 100644 --- a/packages/core/src/v3/taskContext/otelProcessors.ts +++ b/packages/core/src/v3/taskContext/otelProcessors.ts @@ -30,6 +30,12 @@ export class TaskContextSpanProcessor implements SpanProcessor { span.setAttributes( flattenAttributes(taskContext.attributes, SemanticInternalAttributes.METADATA) ); + + // Set run tags as a proper array attribute (not flattened) so it arrives + // as an OTEL ArrayValue and can be extracted on the server side. + if (!taskContext.isRunDisabled && taskContext.ctx.run.tags?.length) { + span.setAttribute(SemanticInternalAttributes.RUN_TAGS, taskContext.ctx.run.tags); + } } if (!isPartialSpan(span) && !skipPartialSpan(span)) { From 72c18795878f84ec4a5869cb0fcaab99872f9e78 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Thu, 12 Mar 2026 10:15:35 +0000 Subject: [PATCH 04/30] New AI streamText span sidebar --- .../components/runs/v3/ai/AIChatMessages.tsx | 253 +++++++++ .../components/runs/v3/ai/AIModelSummary.tsx | 125 +++++ .../components/runs/v3/ai/AISpanDetails.tsx | 202 ++++++++ .../runs/v3/ai/AIToolsInventory.tsx | 83 +++ .../runs/v3/ai/extractAISpanData.ts | 485 ++++++++++++++++++ .../webapp/app/components/runs/v3/ai/index.ts | 3 + .../webapp/app/components/runs/v3/ai/types.ts | 100 ++++ .../app/presenters/v3/SpanPresenter.server.ts | 14 + .../route.tsx | 50 +- apps/webapp/app/tailwind.css | 4 + .../clickhouseEventRepository.server.ts | 31 +- .../app/v3/eventRepository/common.server.ts | 5 +- .../app/v3/llmPricingRegistry.server.ts | 11 - apps/webapp/app/v3/otlpExporter.server.ts | 46 +- .../v3/utils/enrichCreatableEvents.server.ts | 50 -- internal-packages/llm-pricing/src/registry.ts | 11 - 16 files changed, 1377 insertions(+), 96 deletions(-) create mode 100644 apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx create mode 100644 apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx create mode 100644 apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx create mode 100644 apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx create mode 100644 apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts create mode 100644 apps/webapp/app/components/runs/v3/ai/index.ts create mode 100644 apps/webapp/app/components/runs/v3/ai/types.ts diff --git a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx new file mode 100644 index 00000000000..9fe39482c86 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx @@ -0,0 +1,253 @@ +import { lazy, Suspense, useState } from "react"; +import { CodeBlock } from "~/components/code/CodeBlock"; +import type { DisplayItem, ToolUse } from "./types"; + +// Lazy load streamdown to avoid SSR issues +const StreamdownRenderer = lazy(() => + import("streamdown").then((mod) => ({ + default: ({ children }: { children: string }) => ( + + {children} + + ), + })) +); + +export function AIChatMessages({ items }: { items: DisplayItem[] }) { + return ( +
+ {items.map((item, i) => { + switch (item.type) { + case "system": + return ; + case "user": + return ; + case "tool-use": + return ; + case "assistant": + return ; + } + })} +
+ ); +} + +// --------------------------------------------------------------------------- +// Section header (shared across all sections) +// --------------------------------------------------------------------------- + +function SectionHeader({ + label, + right, +}: { + label: string; + right?: React.ReactNode; +}) { + return ( +
+ {label} + {right &&
{right}
} +
+ ); +} + +// --------------------------------------------------------------------------- +// System +// --------------------------------------------------------------------------- + +function SystemSection({ text }: { text: string }) { + const [expanded, setExpanded] = useState(false); + const isLong = text.length > 150; + const preview = isLong ? text.slice(0, 150) + "..." : text; + + return ( +
+ setExpanded(!expanded)} + className="text-[10px] text-text-link hover:underline" + > + {expanded ? "Collapse" : "Expand"} + + ) : undefined + } + /> + 
+        {expanded || !isLong ? text : preview}
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// User +// --------------------------------------------------------------------------- + +function UserSection({ text }: { text: string }) { + return ( +
+ +

{text}

+
+ ); +} + +// --------------------------------------------------------------------------- +// Assistant response (with markdown/raw toggle) +// --------------------------------------------------------------------------- + +export function AssistantResponse({ + text, + headerLabel = "Assistant", +}: { + text: string; + headerLabel?: string; +}) { + const [mode, setMode] = useState<"rendered" | "raw">("rendered"); + + return ( +
+ + + +
+ } + /> + {mode === "rendered" ? ( +
+ {text}}> + {text} + +
+ ) : ( + + )} +
+ ); +} + +// --------------------------------------------------------------------------- +// Tool use (merged calls + results) +// --------------------------------------------------------------------------- + +function ToolUseSection({ tools }: { tools: ToolUse[] }) { + return ( +
+ + {tools.map((tool) => ( + + ))} +
+ ); +} + +type ToolTab = "input" | "output" | "details"; + +function ToolUseRow({ tool }: { tool: ToolUse }) { + const hasInput = tool.inputJson !== "{}"; + const hasResult = !!tool.resultOutput; + const hasDetails = !!tool.description || !!tool.parametersJson; + + const availableTabs: ToolTab[] = [ + ...(hasInput ? (["input"] as const) : []), + ...(hasResult ? (["output"] as const) : []), + ...(hasDetails ? (["details"] as const) : []), + ]; + + const defaultTab: ToolTab | null = hasInput ? "input" : null; + const [activeTab, setActiveTab] = useState(defaultTab); + + function handleTabClick(tab: ToolTab) { + setActiveTab(activeTab === tab ? null : tab); + } + + return ( +
+
+ {tool.toolName} + {tool.resultSummary && ( + {tool.resultSummary} + )} +
+ + {availableTabs.length > 0 && ( + <> +
+ {availableTabs.map((tab) => ( + + ))} +
+ + {activeTab === "input" && hasInput && ( +
+ +
+ )} + + {activeTab === "output" && hasResult && ( +
+ +
+ )} + + {activeTab === "details" && hasDetails && ( +
+ {tool.description && ( +

{tool.description}

+ )} + {tool.parametersJson && ( +
+ + Parameters schema + + +
+ )} +
+ )} + + )} +
+ ); +} diff --git a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx new file mode 100644 index 00000000000..9e627c844d3 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx @@ -0,0 +1,125 @@ +import { formatCurrencyAccurate } from "~/utils/numberFormatter"; +import type { AISpanData } from "./types"; + +export function AITagsRow({ aiData }: { aiData: AISpanData }) { + return ( +
+ {aiData.model} + {aiData.provider !== "unknown" && {aiData.provider}} + {aiData.finishReason && {aiData.finishReason}} + {aiData.serviceTier && tier: {aiData.serviceTier}} + {aiData.toolChoice && tools: {aiData.toolChoice}} + {aiData.toolCount != null && aiData.toolCount > 0 && ( + + {aiData.toolCount} {aiData.toolCount === 1 ? "tool" : "tools"} + + )} + {aiData.messageCount != null && ( + + {aiData.messageCount} {aiData.messageCount === 1 ? "msg" : "msgs"} + + )} + {aiData.telemetryMetadata && + Object.entries(aiData.telemetryMetadata).map(([key, value]) => ( + + {key}: {value} + + ))} +
+ ); +} + +export function AIStatsSummary({ aiData }: { aiData: AISpanData }) { + return ( +
+ Stats + +
+ + + {aiData.cachedTokens != null && aiData.cachedTokens > 0 && ( + + )} + {aiData.reasoningTokens != null && aiData.reasoningTokens > 0 && ( + + )} + + + {aiData.totalCost != null && ( + + )} + {aiData.msToFirstChunk != null && ( + + )} + {aiData.tokensPerSecond != null && ( + + )} +
+
+ ); +} + +function MetricRow({ + label, + value, + unit, + bold, + border, +}: { + label: string; + value: string; + unit?: string; + bold?: boolean; + border?: boolean; +}) { + return ( +
+ {label} + + {value} + {unit && {unit}} + +
+ ); +} + +function formatTtfc(ms: number): string { + if (ms >= 10_000) { + return `${(ms / 1000).toFixed(1)}s`; + } + return `${Math.round(ms)}ms`; +} + +function Pill({ + children, + variant = "default", +}: { + children: React.ReactNode; + variant?: "default" | "dimmed"; +}) { + return ( + + {children} + + ); +} diff --git a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx new file mode 100644 index 00000000000..988de788167 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx @@ -0,0 +1,202 @@ +import { useState } from "react"; +import { Clipboard, ClipboardCheck } from "lucide-react"; +import { TabButton, TabContainer } from "~/components/primitives/Tabs"; +import type { AISpanData, DisplayItem } from "./types"; +import { AITagsRow, AIStatsSummary } from "./AIModelSummary"; +import { AIChatMessages, AssistantResponse } from "./AIChatMessages"; +import { AIToolsInventory } from "./AIToolsInventory"; + +type AITab = "overview" | "messages" | "tools"; + +export function AISpanDetails({ + aiData, + rawProperties, +}: { + aiData: AISpanData; + rawProperties?: string; +}) { + const [tab, setTab] = useState("overview"); + const hasTools = + (aiData.toolDefinitions && aiData.toolDefinitions.length > 0) || aiData.toolCount != null; + + return ( +
+ {/* Tab bar */} +
+ + setTab("overview")} + shortcut={{ key: "o" }} + > + Overview + + setTab("messages")} + shortcut={{ key: "m" }} + > + Messages + + {hasTools && ( + setTab("tools")} + shortcut={{ key: "t" }} + > + Tools{aiData.toolCount != null ? ` (${aiData.toolCount})` : ""} + + )} + +
+ + {/* Tab content */} +
+ {tab === "overview" && } + {tab === "messages" && } + {tab === "tools" && } +
+ + {/* Footer: Copy raw */} + {rawProperties && } +
+ ); +} + +function OverviewTab({ aiData }: { aiData: AISpanData }) { + const { userText, outputText, outputToolNames } = extractInputOutput(aiData); + + return ( +
+ {/* Tags + Stats */} + + + + {/* Input (last user prompt) */} + {userText && ( +
+ + Input + +

{userText}

+
+ )} + + {/* Output (assistant response or tool calls) */} + {outputText && } + {outputToolNames.length > 0 && !outputText && ( +
+ + Output + +

+ Called {outputToolNames.length === 1 ? "tool" : "tools"}:{" "} + {outputToolNames.join(", ")} +

+
+ )} +
+ ); +} + +function MessagesTab({ aiData }: { aiData: AISpanData }) { + return ( +
+
+ {aiData.items && aiData.items.length > 0 && } + {aiData.responseText && !hasAssistantItem(aiData.items) && ( + + )} +
+
+ ); +} + +function ToolsTab({ aiData }: { aiData: AISpanData }) { + return ; +} + +function CopyRawFooter({ rawProperties }: { rawProperties: string }) { + const [copied, setCopied] = useState(false); + + function handleCopy() { + navigator.clipboard.writeText(rawProperties); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } + + return ( +
+ +
+ ); +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function extractInputOutput(aiData: AISpanData): { + userText: string | undefined; + outputText: string | undefined; + outputToolNames: string[]; +} { + let userText: string | undefined; + let outputText: string | undefined; + const outputToolNames: string[] = []; + + if (aiData.items) { + // Find the last user message + for (let i = aiData.items.length - 1; i >= 0; i--) { + if (aiData.items[i].type === "user") { + userText = (aiData.items[i] as { type: "user"; text: string }).text; + break; + } + } + + // Find the last assistant or tool-use item as the output + for (let i = aiData.items.length - 1; i >= 0; i--) { + const item = aiData.items[i]; + if (item.type === "assistant") { + outputText = item.text; + break; + } + if (item.type === "tool-use") { + for (const tool of item.tools) { + outputToolNames.push(tool.toolName); + } + break; + } + } + } + + // Fall back to responseText if no assistant item found + if (!outputText && aiData.responseText) { + outputText = aiData.responseText; + } + + return { userText, outputText, outputToolNames }; +} + +function hasAssistantItem(items: DisplayItem[] | undefined): boolean { + if (!items) return false; + return items.some((item) => item.type === "assistant"); +} diff --git a/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx b/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx new file mode 100644 index 00000000000..5130256f85b --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx @@ -0,0 +1,83 @@ +import { useState } from "react"; +import { CodeBlock } from "~/components/code/CodeBlock"; +import type { AISpanData, ToolDefinition } from "./types"; + +export function AIToolsInventory({ aiData }: { aiData: AISpanData }) { + const defs = aiData.toolDefinitions ?? []; + const calledNames = getCalledToolNames(aiData); + + if (defs.length === 0) { + return ( +
+ No tool definitions available for this span. +
+ ); + } + + return ( +
+ {defs.map((def) => { + const wasCalled = calledNames.has(def.name); + return ; + })} +
+ ); +} + +function ToolDefRow({ def, wasCalled }: { def: ToolDefinition; wasCalled: boolean }) { + const [showSchema, setShowSchema] = useState(false); + + return ( +
+
+
+ {def.name} + {wasCalled ? "called" : "not called"} +
+ + {def.description && ( +

{def.description}

+ )} + + {def.parametersJson && ( +
+ + {showSchema && ( +
+ +
+ )} +
+ )} +
+ ); +} + +function getCalledToolNames(aiData: AISpanData): Set { + const names = new Set(); + if (!aiData.items) return names; + + for (const item of aiData.items) { + if (item.type === "tool-use") { + for (const tool of item.tools) { + names.add(tool.toolName); + } + } + } + + return names; +} diff --git a/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts b/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts new file mode 100644 index 00000000000..c9f0562afa4 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts @@ -0,0 +1,485 @@ +import type { AISpanData, DisplayItem, ToolDefinition, ToolUse } from "./types"; + +/** + * Extracts structured AI span data from unflattened OTEL span properties. + * + * Works with the nested object produced by `unflattenAttributes()` — expects + * keys like `gen_ai.response.model`, `ai.prompt.messages`, `trigger.llm.total_cost`, etc. + * + * @param properties Unflattened span properties object + * @param durationMs Span duration in milliseconds + * @returns Structured AI data, or undefined if this isn't an AI generation span + */ +export function extractAISpanData( + properties: Record, + durationMs: number +): AISpanData | undefined { + const genAi = properties.gen_ai; + if (!genAi || typeof genAi !== "object") return undefined; + + const g = genAi as Record; + const ai = rec(properties.ai); + const trigger = rec(properties.trigger); + + const gResponse = rec(g.response); + const gRequest = rec(g.request); + const gUsage = rec(g.usage); + const gOperation = rec(g.operation); + const aiModel = rec(ai.model); + const aiResponse = rec(ai.response); + const aiPrompt = rec(ai.prompt); + const aiUsage = rec(ai.usage); + const triggerLlm = rec(trigger.llm); + + const model = str(gResponse.model) ?? str(gRequest.model) ?? str(aiModel.id); + if (!model) return undefined; + + // Prefer ai.usage (richer) over gen_ai.usage + const inputTokens = num(aiUsage.inputTokens) ?? num(gUsage.input_tokens) ?? 0; + const outputTokens = num(aiUsage.outputTokens) ?? num(gUsage.output_tokens) ?? 0; + const totalTokens = num(aiUsage.totalTokens) ?? inputTokens + outputTokens; + + const tokensPerSecond = + num(aiResponse.avgOutputTokensPerSecond) ?? + (outputTokens > 0 && durationMs > 0 + ? Math.round((outputTokens / (durationMs / 1000)) * 10) / 10 + : undefined); + + const toolDefs = parseToolDefinitions(aiPrompt.tools); + const providerMeta = parseProviderMetadata(aiResponse.providerMetadata); + const aiTelemetry = rec(ai.telemetry); + const telemetryMeta = extractTelemetryMetadata(aiTelemetry.metadata); + + return { + model, + provider: str(g.system) ?? "unknown", + operationName: str(gOperation.name) ?? str(ai.operationId) ?? "", + finishReason: str(aiResponse.finishReason), + serviceTier: providerMeta?.serviceTier, + toolChoice: parseToolChoice(aiPrompt.toolChoice), + toolCount: toolDefs?.length, + messageCount: countMessages(aiPrompt.messages), + telemetryMetadata: telemetryMeta, + inputTokens, + outputTokens, + totalTokens, + cachedTokens: num(aiUsage.cachedInputTokens) ?? num(gUsage.cache_read_input_tokens), + reasoningTokens: num(aiUsage.reasoningTokens) ?? num(gUsage.reasoning_tokens), + tokensPerSecond, + msToFirstChunk: num(aiResponse.msToFirstChunk), + durationMs, + inputCost: num(triggerLlm.input_cost), + outputCost: num(triggerLlm.output_cost), + totalCost: num(triggerLlm.total_cost), + responseText: str(aiResponse.text) || undefined, + toolDefinitions: toolDefs, + items: buildDisplayItems(aiPrompt.messages, aiResponse.toolCalls, toolDefs), + }; +} + +// --------------------------------------------------------------------------- +// Primitive helpers +// --------------------------------------------------------------------------- + +function rec(v: unknown): Record { + return v && typeof v === "object" ? (v as Record) : {}; +} + +function str(v: unknown): string | undefined { + return typeof v === "string" ? v : undefined; +} + +function num(v: unknown): number | undefined { + return typeof v === "number" ? v : undefined; +} + +// --------------------------------------------------------------------------- +// Message → DisplayItem transformation +// --------------------------------------------------------------------------- + +type RawMessage = { + role: string; + content: unknown; + toolCallId?: string; + name?: string; +}; + +/** + * Build display items from prompt messages and optionally response tool calls. + * - Parses ai.prompt.messages and merges consecutive tool-call + tool-result pairs + * - If ai.response.toolCalls is present (finishReason=tool-calls), appends those too + */ +function buildDisplayItems( + messagesRaw: unknown, + responseToolCallsRaw: unknown, + toolDefs?: ToolDefinition[] +): DisplayItem[] | undefined { + const items = parseMessagesToDisplayItems(messagesRaw); + const responseToolCalls = parseResponseToolCalls(responseToolCallsRaw); + + if (!items && !responseToolCalls) return undefined; + + const result = items ?? []; + + if (responseToolCalls && responseToolCalls.length > 0) { + result.push({ type: "tool-use", tools: responseToolCalls }); + } + + if (toolDefs && toolDefs.length > 0) { + const defsByName = new Map(toolDefs.map((d) => [d.name, d])); + for (const item of result) { + if (item.type === "tool-use") { + for (const tool of item.tools) { + const def = defsByName.get(tool.toolName); + if (def) { + tool.description = def.description; + tool.parametersJson = def.parametersJson; + } + } + } + } + } + + return result.length > 0 ? result : undefined; +} + +function parseMessagesToDisplayItems(raw: unknown): DisplayItem[] | undefined { + if (typeof raw !== "string") return undefined; + + let messages: RawMessage[]; + try { + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return undefined; + messages = parsed.map((item: unknown) => { + const m = rec(item); + return { + role: str(m.role) ?? "user", + content: m.content, + toolCallId: str(m.toolCallId), + name: str(m.name), + }; + }); + } catch { + return undefined; + } + + const items: DisplayItem[] = []; + let i = 0; + + while (i < messages.length) { + const msg = messages[i]; + + if (msg.role === "system") { + items.push({ type: "system", text: extractTextContent(msg.content) }); + i++; + continue; + } + + if (msg.role === "user") { + items.push({ type: "user", text: extractTextContent(msg.content) }); + i++; + continue; + } + + // Assistant message — check if it contains tool calls + if (msg.role === "assistant") { + const toolCalls = extractToolCalls(msg.content); + + if (toolCalls.length > 0) { + // Collect subsequent tool result messages that match these tool calls + const toolCallIds = new Set(toolCalls.map((tc) => tc.toolCallId)); + let j = i + 1; + while (j < messages.length && messages[j].role === "tool") { + j++; + } + // Gather tool result messages between i+1 and j + const toolResultMsgs = messages.slice(i + 1, j); + + // Build ToolUse entries by pairing calls with results + const tools: ToolUse[] = toolCalls.map((tc) => { + const resultMsg = toolResultMsgs.find((m) => { + // Match by toolCallId in the message's content parts + const results = extractToolResults(m.content); + return results.some((r) => r.toolCallId === tc.toolCallId); + }); + + const result = resultMsg + ? extractToolResults(resultMsg.content).find( + (r) => r.toolCallId === tc.toolCallId + ) + : undefined; + + return { + toolCallId: tc.toolCallId, + toolName: tc.toolName, + inputJson: JSON.stringify(tc.input, null, 2), + resultSummary: result?.summary, + resultOutput: result?.formattedOutput, + }; + }); + + items.push({ type: "tool-use", tools }); + i = j; // skip past the tool result messages + continue; + } + + // Assistant message with just text + const text = extractTextContent(msg.content); + if (text) { + items.push({ type: "assistant", text }); + } + i++; + continue; + } + + // Skip any other message types (tool messages that weren't consumed above) + i++; + } + + return items.length > 0 ? items : undefined; +} + +// --------------------------------------------------------------------------- +// Response tool calls (from ai.response.toolCalls, used when finishReason=tool-calls) +// --------------------------------------------------------------------------- + +/** + * Parse ai.response.toolCalls JSON string into ToolUse entries. + * These are tool calls the model requested but haven't been executed yet in this span. + */ +function parseResponseToolCalls(raw: unknown): ToolUse[] | undefined { + if (typeof raw !== "string") return undefined; + try { + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return undefined; + const tools: ToolUse[] = []; + for (const item of parsed) { + const tc = rec(item); + if (tc.type === "tool-call" || tc.toolName || tc.toolCallId) { + tools.push({ + toolCallId: str(tc.toolCallId) ?? "", + toolName: str(tc.toolName) ?? "", + inputJson: JSON.stringify( + tc.input && typeof tc.input === "object" ? tc.input : {}, + null, + 2 + ), + }); + } + } + return tools.length > 0 ? tools : undefined; + } catch { + return undefined; + } +} + +// --------------------------------------------------------------------------- +// Content part extraction +// --------------------------------------------------------------------------- + +function extractTextContent(content: unknown): string { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + + const texts: string[] = []; + for (const raw of content) { + const p = rec(raw); + if (p.type === "text" && typeof p.text === "string") { + texts.push(p.text); + } else if (typeof p.text === "string") { + texts.push(p.text); + } + } + return texts.join("\n"); +} + +type ParsedToolCall = { + toolCallId: string; + toolName: string; + input: Record; +}; + +function extractToolCalls(content: unknown): ParsedToolCall[] { + if (!Array.isArray(content)) return []; + const calls: ParsedToolCall[] = []; + for (const raw of content) { + const p = rec(raw); + if (p.type === "tool-call") { + calls.push({ + toolCallId: str(p.toolCallId) ?? "", + toolName: str(p.toolName) ?? "", + input: p.input && typeof p.input === "object" ? (p.input as Record) : {}, + }); + } + } + return calls; +} + +type ParsedToolResult = { + toolCallId: string; + toolName: string; + summary: string; + formattedOutput: string; +}; + +function extractToolResults(content: unknown): ParsedToolResult[] { + if (!Array.isArray(content)) return []; + const results: ParsedToolResult[] = []; + for (const raw of content) { + const p = rec(raw); + if (p.type === "tool-result") { + const { summary, formattedOutput } = summarizeToolOutput(p.output); + results.push({ + toolCallId: str(p.toolCallId) ?? "", + toolName: str(p.toolName) ?? "", + summary, + formattedOutput, + }); + } + } + return results; +} + +/** + * Summarize a tool output into a short label and a formatted string for display. + * Handles the AI SDK's `{ type: "json", value: { status, contentType, body, truncated } }` shape. + */ +function summarizeToolOutput(output: unknown): { summary: string; formattedOutput: string } { + if (typeof output === "string") { + return { + summary: output.length > 80 ? output.slice(0, 80) + "..." : output, + formattedOutput: output, + }; + } + + if (!output || typeof output !== "object") { + return { summary: "result", formattedOutput: JSON.stringify(output, null, 2) }; + } + + const o = output as Record; + + // AI SDK wraps tool results as { type: "json", value: { status, contentType, body, ... } } + if (o.type === "json" && o.value && typeof o.value === "object") { + const v = o.value as Record; + const parts: string[] = []; + if (typeof v.status === "number") parts.push(`${v.status}`); + if (typeof v.contentType === "string") parts.push(v.contentType); + if (v.truncated === true) parts.push("truncated"); + return { + summary: parts.length > 0 ? parts.join(" · ") : "json result", + formattedOutput: JSON.stringify(v, null, 2), + }; + } + + return { summary: "result", formattedOutput: JSON.stringify(output, null, 2) }; +} + +// --------------------------------------------------------------------------- +// Tool definitions (from ai.prompt.tools) +// --------------------------------------------------------------------------- + +/** + * Parse ai.prompt.tools — after the array fix, this arrives as a JSON array string + * where each element is itself a JSON string of a tool definition. + */ +function parseToolDefinitions(raw: unknown): ToolDefinition[] | undefined { + if (typeof raw !== "string") return undefined; + try { + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return undefined; + const defs: ToolDefinition[] = []; + for (const item of parsed) { + // Each item is either a JSON string or already an object + const obj = typeof item === "string" ? JSON.parse(item) : item; + if (!obj || typeof obj !== "object") continue; + const o = obj as Record; + const name = str(o.name); + if (!name) continue; + const schema = o.parameters ?? o.inputSchema; + defs.push({ + name, + description: str(o.description), + parametersJson: + schema && typeof schema === "object" + ? JSON.stringify(schema, null, 2) + : undefined, + }); + } + return defs.length > 0 ? defs : undefined; + } catch { + return undefined; + } +} + +// --------------------------------------------------------------------------- +// Provider metadata (service tier, inference geo, etc.) +// --------------------------------------------------------------------------- + +function parseProviderMetadata( + raw: unknown +): { serviceTier?: string } | undefined { + if (typeof raw !== "string") return undefined; + try { + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== "object") return undefined; + + // Anthropic: { anthropic: { usage: { service_tier: "standard" } } } + const anthropic = rec(parsed.anthropic ?? parsed); + const usage = rec(anthropic.usage); + const serviceTier = str(usage.service_tier); + + return serviceTier ? { serviceTier } : undefined; + } catch { + return undefined; + } +} + +// --------------------------------------------------------------------------- +// Tool choice parsing +// --------------------------------------------------------------------------- + +function parseToolChoice(raw: unknown): string | undefined { + if (typeof raw !== "string") return undefined; + try { + const parsed = JSON.parse(raw); + if (typeof parsed === "string") return parsed; + if (parsed && typeof parsed === "object" && typeof parsed.type === "string") { + return parsed.type; + } + return undefined; + } catch { + return raw || undefined; + } +} + +// --------------------------------------------------------------------------- +// Message count +// --------------------------------------------------------------------------- + +function countMessages(raw: unknown): number | undefined { + if (typeof raw !== "string") return undefined; + try { + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return undefined; + return parsed.length > 0 ? parsed.length : undefined; + } catch { + return undefined; + } +} + +// --------------------------------------------------------------------------- +// Telemetry metadata +// --------------------------------------------------------------------------- + +function extractTelemetryMetadata(raw: unknown): Record | undefined { + if (!raw || typeof raw !== "object") return undefined; + + const result: Record = {}; + for (const [key, value] of Object.entries(raw as Record)) { + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + result[key] = String(value); + } + } + + return Object.keys(result).length > 0 ? result : undefined; +} diff --git a/apps/webapp/app/components/runs/v3/ai/index.ts b/apps/webapp/app/components/runs/v3/ai/index.ts new file mode 100644 index 00000000000..7e33a46fb2c --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/index.ts @@ -0,0 +1,3 @@ +export { AISpanDetails } from "./AISpanDetails"; +export { extractAISpanData } from "./extractAISpanData"; +export type { AISpanData, DisplayItem, ToolUse } from "./types"; diff --git a/apps/webapp/app/components/runs/v3/ai/types.ts b/apps/webapp/app/components/runs/v3/ai/types.ts new file mode 100644 index 00000000000..11fdce4e606 --- /dev/null +++ b/apps/webapp/app/components/runs/v3/ai/types.ts @@ -0,0 +1,100 @@ +// --------------------------------------------------------------------------- +// Tool use (merged assistant tool-call + tool result) +// --------------------------------------------------------------------------- + +export type ToolDefinition = { + name: string; + description?: string; + /** JSON schema as formatted string */ + parametersJson?: string; +}; + +export type ToolUse = { + toolCallId: string; + toolName: string; + /** Tool description from the definition, if available */ + description?: string; + /** JSON schema of the tool's parameters, pretty-printed */ + parametersJson?: string; + /** Formatted input args as JSON string */ + inputJson: string; + /** Short summary of the result (e.g. "200 · text/html · truncated") */ + resultSummary?: string; + /** Full formatted result for display in a code block */ + resultOutput?: string; +}; + +// --------------------------------------------------------------------------- +// Display items — what the UI actually renders +// --------------------------------------------------------------------------- + +/** System prompt text (collapsible) */ +export type SystemItem = { + type: "system"; + text: string; +}; + +/** User message text */ +export type UserItem = { + type: "user"; + text: string; +}; + +/** One or more tool calls with their results, grouped */ +export type ToolUseItem = { + type: "tool-use"; + tools: ToolUse[]; +}; + +/** Final assistant text response */ +export type AssistantItem = { + type: "assistant"; + text: string; +}; + +export type DisplayItem = SystemItem | UserItem | ToolUseItem | AssistantItem; + +// --------------------------------------------------------------------------- +// Span-level AI data +// --------------------------------------------------------------------------- + +export type AISpanData = { + model: string; + provider: string; + operationName: string; + + // Categorical tags + finishReason?: string; + serviceTier?: string; + toolChoice?: string; + toolCount?: number; + messageCount?: number; + /** User-defined telemetry metadata (from ai.telemetry.metadata) */ + telemetryMetadata?: Record; + + // Token counts + inputTokens: number; + outputTokens: number; + totalTokens: number; + cachedTokens?: number; + reasoningTokens?: number; + + // Performance + tokensPerSecond?: number; + msToFirstChunk?: number; + durationMs: number; + + // Cost + inputCost?: number; + outputCost?: number; + totalCost?: number; + + // Response text (final assistant output) + responseText?: string; + + // Tool definitions (from ai.prompt.tools) + toolDefinitions?: ToolDefinition[]; + + // Display-ready message items (system, user, tool-use groups, assistant text) + items?: DisplayItem[]; +}; diff --git a/apps/webapp/app/presenters/v3/SpanPresenter.server.ts b/apps/webapp/app/presenters/v3/SpanPresenter.server.ts index ce83c2e242b..9ad94745616 100644 --- a/apps/webapp/app/presenters/v3/SpanPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/SpanPresenter.server.ts @@ -24,6 +24,7 @@ import { engine } from "~/v3/runEngine.server"; import { resolveEventRepositoryForStore } from "~/v3/eventRepository/index.server"; import { IEventRepository, SpanDetail } from "~/v3/eventRepository/eventRepository.types"; import { safeJsonParse } from "~/utils/json"; +import { extractAISpanData } from "~/components/runs/v3/ai"; type Result = Awaited>; export type Span = NonNullable["span"]>; @@ -543,6 +544,13 @@ export class SpanPresenter extends BasePresenter { entity: span.entity, metadata: span.metadata, triggeredRuns, + aiData: + span.properties && typeof span.properties === "object" + ? extractAISpanData( + span.properties as Record, + span.duration / 1_000_000 + ) + : undefined, }; switch (span.entity.type) { @@ -665,6 +673,12 @@ export class SpanPresenter extends BasePresenter { }; } default: + if (data.aiData) { + return { + ...data, + entity: { type: "ai-generation" as const, object: data.aiData }, + }; + } return { ...data, entity: null }; } } diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx index e46eaa5148f..8f7ae61b5d5 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx @@ -19,7 +19,7 @@ import { taskRunErrorEnhancer, } from "@trigger.dev/core/v3"; import { assertNever } from "assert-never"; -import { useEffect } from "react"; +import { useEffect, useState } from "react"; import { typedjson, useTypedFetcher } from "remix-typedjson"; import { ExitIcon } from "~/assets/icons/ExitIcon"; import { FlagIcon } from "~/assets/icons/RegionIcons"; @@ -60,6 +60,7 @@ import { RunIcon } from "~/components/runs/v3/RunIcon"; import { RunTag } from "~/components/runs/v3/RunTag"; import { TruncatedCopyableValue } from "~/components/primitives/TruncatedCopyableValue"; import { SpanEvents } from "~/components/runs/v3/SpanEvents"; +import { AISpanDetails } from "~/components/runs/v3/ai"; import { SpanTitle } from "~/components/runs/v3/SpanTitle"; import { TaskRunAttemptStatusCombo } from "~/components/runs/v3/TaskRunAttemptStatus"; import { @@ -252,6 +253,8 @@ function SpanBody({ span = applySpanOverrides(span, spanOverrides); + const isAiGeneration = span.entity?.type === "ai-generation"; + return (
@@ -276,9 +279,13 @@ function SpanBody({ /> )}
-
+ {isAiGeneration ? ( -
+ ) : ( +
+ +
+ )}
); } @@ -1155,6 +1162,35 @@ function RunError({ error }: { error: TaskRunError }) { } } +function CollapsibleProperties({ code }: { code: string }) { + const [open, setOpen] = useState(false); + return ( +
+ + {open && ( +
+ +
+ )} +
+ ); +} + function SpanEntity({ span }: { span: Span }) { const isAdmin = useHasAdminAccess(); @@ -1352,6 +1388,14 @@ function SpanEntity({ span }: { span: Span }) { /> ); } + case "ai-generation": { + return ( + + ); + } default: { assertNever(span.entity); } diff --git a/apps/webapp/app/tailwind.css b/apps/webapp/app/tailwind.css index 2e435a032b7..8ec29b91568 100644 --- a/apps/webapp/app/tailwind.css +++ b/apps/webapp/app/tailwind.css @@ -85,6 +85,10 @@ } @layer utilities { + .scrollbar-gutter-stable { + scrollbar-gutter: stable; + } + .animated-gradient-glow { position: relative; overflow: visible; diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts index 7a46b6b3bfb..531c3e307e9 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts @@ -8,9 +8,7 @@ import type { TaskEventV2Input, } from "@internal/clickhouse"; import { Attributes, startSpan, trace, Tracer } from "@internal/tracing"; -import { trail } from "agentcrumbs"; // @crumbs -const crumb = trail("webapp:llm-dual-write"); // @crumbs import { createJsonErrorObject } from "@trigger.dev/core/v3/errors"; import { serializeTraceparent } from "@trigger.dev/core/v3/isomorphic"; import { @@ -233,7 +231,6 @@ export class ClickhouseEventRepository implements IEventRepository { } async #flushLlmUsageBatch(flushId: string, rows: LlmUsageV1Input[]) { - crumb("flushing llm usage batch", { flushId, rows: rows.length }); // @crumbs const [insertError] = await this._clickhouse.llmUsage.insert(rows, { params: { @@ -242,12 +239,9 @@ export class ClickhouseEventRepository implements IEventRepository { }); if (insertError) { - crumb("llm usage batch insert failed", { flushId, error: String(insertError) }); // @crumbs throw insertError; } - crumb("llm usage batch inserted", { flushId, rows: rows.length }); // @crumbs - logger.info("ClickhouseEventRepository.flushLlmUsageBatch Inserted LLM usage batch", { rows: rows.length, }); @@ -312,7 +306,6 @@ export class ClickhouseEventRepository implements IEventRepository { .map((e) => this.#createLlmUsageInput(e)); if (llmUsageRows.length > 0) { - crumb("queuing llm usage rows", { count: llmUsageRows.length, firstRunId: llmUsageRows[0]?.run_id, metadataKeys: llmUsageRows.map((r) => Object.keys(r.metadata)) }); // @crumbs this._llmUsageFlushScheduler.addToBatch(llmUsageRows); } } @@ -1381,19 +1374,21 @@ export class ClickhouseEventRepository implements IEventRepository { } } - if ( - (span.properties == null || - (typeof span.properties === "object" && Object.keys(span.properties).length === 0)) && - typeof record.attributes_text === "string" - ) { - const parsedAttributes = this.#parseAttributes(record.attributes_text); - const resourceAttributes = parsedAttributes["$resource"]; + if (typeof record.attributes_text === "string") { + const shouldUpdate = + span.properties == null || + (typeof span.properties === "object" && Object.keys(span.properties).length === 0) || + (record.kind === "SPAN" && record.status !== "PARTIAL"); - // Remove the $resource key from the attributes - delete parsedAttributes["$resource"]; + if (shouldUpdate) { + const parsedAttributes = this.#parseAttributes(record.attributes_text); + const resourceAttributes = parsedAttributes["$resource"]; - span.properties = parsedAttributes; - span.resourceProperties = resourceAttributes as Record | undefined; + delete parsedAttributes["$resource"]; + + span.properties = parsedAttributes; + span.resourceProperties = resourceAttributes as Record | undefined; + } } } diff --git a/apps/webapp/app/v3/eventRepository/common.server.ts b/apps/webapp/app/v3/eventRepository/common.server.ts index 2e3bdf37c50..3ba8a50c7f7 100644 --- a/apps/webapp/app/v3/eventRepository/common.server.ts +++ b/apps/webapp/app/v3/eventRepository/common.server.ts @@ -140,7 +140,8 @@ export function createExceptionPropertiesFromError(error: TaskRunError): Excepti } } -// removes keys that start with a $ sign. If there are no keys left, return undefined +// Removes internal/private attribute keys from span properties. +// Filters: "$" prefixed keys (private metadata) and "ctx." prefixed keys (Trigger.dev run context) export function removePrivateProperties( attributes: Attributes | undefined | null ): Attributes | undefined { @@ -151,7 +152,7 @@ export function removePrivateProperties( const result: Attributes = {}; for (const [key, value] of Object.entries(attributes)) { - if (key.startsWith("$")) { + if (key.startsWith("$") || key.startsWith("ctx.")) { continue; } diff --git a/apps/webapp/app/v3/llmPricingRegistry.server.ts b/apps/webapp/app/v3/llmPricingRegistry.server.ts index d05b1afc2c8..e90a84689f5 100644 --- a/apps/webapp/app/v3/llmPricingRegistry.server.ts +++ b/apps/webapp/app/v3/llmPricingRegistry.server.ts @@ -1,37 +1,28 @@ import { ModelPricingRegistry, seedLlmPricing } from "@internal/llm-pricing"; -import { trail } from "agentcrumbs"; // @crumbs import { prisma, $replica } from "~/db.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; import { setLlmPricingRegistry } from "./utils/enrichCreatableEvents.server"; -const crumb = trail("webapp:llm-registry"); // @crumbs - async function initRegistry(registry: ModelPricingRegistry) { if (env.LLM_PRICING_SEED_ON_STARTUP) { - crumb("seeding llm pricing on startup"); // @crumbs const result = await seedLlmPricing(prisma); - crumb("seed complete", { modelsCreated: result.modelsCreated, modelsSkipped: result.modelsSkipped }); // @crumbs } await registry.loadFromDatabase(); - crumb("registry loaded successfully", { isLoaded: registry.isLoaded }); // @crumbs } export const llmPricingRegistry = singleton("llmPricingRegistry", () => { if (!env.LLM_COST_TRACKING_ENABLED) { - crumb("llm cost tracking disabled via env"); // @crumbs return null; } - crumb("initializing registry singleton"); // @crumbs const registry = new ModelPricingRegistry($replica); // Wire up the registry so enrichCreatableEvents can use it setLlmPricingRegistry(registry); initRegistry(registry).catch((err) => { - crumb("registry init failed", { error: String(err) }); // @crumbs console.error("Failed to initialize LLM pricing registry", err); }); @@ -41,10 +32,8 @@ export const llmPricingRegistry = singleton("llmPricingRegistry", () => { registry .reload() .then(() => { - crumb("registry reloaded"); // @crumbs }) .catch((err) => { - crumb("registry reload failed", { error: String(err) }); // @crumbs console.error("Failed to reload LLM pricing registry", err); }); }, reloadInterval); diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index b5b5fee70fc..0a86fb65ec9 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -395,7 +395,22 @@ function convertSpansToCreateableEvents( ); const runTags = extractArrayAttribute(span.attributes ?? [], SemanticInternalAttributes.RUN_TAGS); - if (runTags && runTags.length > 0) { crumbOtlp("extracted runTags from span", { runTags, spanId: binaryToHex(span.spanId) }); } // @crumbs + + // #region @crumbs + if (span.attributes) { + crumbOtlp("span raw OTEL attrs", { + spanName: span.name, + spanId: binaryToHex(span.spanId), + attrCount: span.attributes.length, + attrs: span.attributes.map((a) => ({ + key: a.key, + type: a.value?.stringValue !== undefined ? "string" : a.value?.intValue !== undefined ? "int" : a.value?.doubleValue !== undefined ? "double" : a.value?.boolValue !== undefined ? "bool" : a.value?.arrayValue ? "array" : a.value?.bytesValue ? "bytes" : "unknown", + ...(a.value?.arrayValue ? { arrayLen: a.value.arrayValue.values?.length } : {}), + ...(a.value?.stringValue !== undefined ? { strLen: a.value.stringValue.length } : {}), + })), + }); + } + // #endregion @crumbs const properties = truncateAttributes( @@ -715,6 +730,8 @@ function convertKeyValueItemsToMap( ? attribute.value.boolValue : isBytesValue(attribute.value) ? binaryToHex(attribute.value.bytesValue) + : isArrayValue(attribute.value) + ? serializeArrayValue(attribute.value.arrayValue!.values) : undefined; return map; @@ -751,6 +768,8 @@ function convertSelectedKeyValueItemsToMap( ? attribute.value.boolValue : isBytesValue(attribute.value) ? binaryToHex(attribute.value.bytesValue) + : isArrayValue(attribute.value) + ? serializeArrayValue(attribute.value.arrayValue!.values) : undefined; return map; @@ -1064,6 +1083,31 @@ function isBytesValue(value: AnyValue | undefined): value is { bytesValue: Buffe return Buffer.isBuffer(value.bytesValue); } +function isArrayValue( + value: AnyValue | undefined +): value is { arrayValue: { values: AnyValue[] } } { + if (!value) return false; + + return value.arrayValue != null && Array.isArray(value.arrayValue.values); +} + +/** + * Serialize an OTEL array value into a JSON string. + * For arrays of strings, produces a JSON array: `["item1","item2"]` + * For mixed types, extracts primitives and serializes. + */ +function serializeArrayValue(values: AnyValue[]): string { + const items = values.map((v) => { + if (isStringValue(v)) return v.stringValue; + if (isIntValue(v)) return Number(v.intValue); + if (isDoubleValue(v)) return v.doubleValue; + if (isBoolValue(v)) return v.boolValue; + return null; + }); + + return JSON.stringify(items); +} + function binaryToHex(buffer: Buffer | string): string; function binaryToHex(buffer: Buffer | string | undefined): string | undefined; function binaryToHex(buffer: Buffer | string | undefined): string | undefined { diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index d76fdda5879..1c125060ae2 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -1,8 +1,5 @@ -import { trail } from "agentcrumbs"; // @crumbs import type { CreateEventInput, LlmUsageData } from "../eventRepository/eventRepository.types"; -const crumb = trail("webapp:llm-enrich"); // @crumbs - // Registry interface — matches ModelPricingRegistry from @internal/llm-pricing type CostRegistry = { isLoaded: boolean; @@ -48,25 +45,6 @@ function enrichLlmCost(event: CreateEventInput): void { const props = event.properties; if (!props) return; - // #region @crumbs - // Log all spans (not just gen_ai) that have conversation/chat/session/user context - if (!event.isPartial) { - const contextKeys = Object.entries(props).filter(([k]) => - k.startsWith("ai.telemetry.") || k.startsWith("gen_ai.conversation") || - k.startsWith("chat.") || k.includes("session") || k.includes("user") - ); - if (contextKeys.length > 0) { - crumb("span with context", { - spanId: event.spanId, - parentId: event.parentId, - runId: event.runId, - message: event.message, - contextAttrs: Object.fromEntries(contextKeys), - }); - } - } - // #endregion @crumbs - // Only enrich span-like events (INTERNAL, SERVER, CLIENT, CONSUMER, PRODUCER — not LOG, UNSPECIFIED) const enrichableKinds = new Set(["INTERNAL", "SERVER", "CLIENT", "CONSUMER", "PRODUCER"]); if (!enrichableKinds.has(event.kind as string)) return; @@ -86,38 +64,25 @@ function enrichLlmCost(event: CreateEventInput): void { : null; if (!responseModel) { - // #region @crumbs - const genAiModel = props["gen_ai.response.model"]; - const aiModel = props["ai.model.id"]; - if (genAiModel || aiModel) { - crumb("responseModel null despite gen_ai attrs", { genAiModel, genAiModelType: typeof genAiModel, aiModel, spanId: event.spanId }); - } - // #endregion @crumbs return; } - crumb("llm span detected", { responseModel, kind: event.kind, spanId: event.spanId }); // @crumbs - // Extract usage details, normalizing attribute names const usageDetails = extractUsageDetails(props); // Need at least some token usage const hasTokens = Object.values(usageDetails).some((v) => v > 0); if (!hasTokens) { - crumb("llm span skipped: no tokens", { responseModel, spanId: event.spanId }); // @crumbs return; } if (!_registry?.isLoaded) { - crumb("llm span skipped: registry not loaded", { responseModel }); // @crumbs return; } const cost = _registry.calculateCost(responseModel, usageDetails); if (!cost) return; - crumb("llm cost enriched", { responseModel, totalCost: cost.totalCost, matchedModel: cost.matchedModelName, spanId: event.spanId }); // @crumbs - // Add trigger.llm.* attributes to the span event.properties = { ...props, @@ -165,21 +130,6 @@ function enrichLlmCost(event: CreateEventInput): void { } } - // #region @crumbs - const metadataKeyCount = Object.keys(metadata).length; - if (metadataKeyCount > 0) { - crumb("llm metadata built", { - spanId: event.spanId, - runId: event.runId, - responseModel, - metadataKeyCount, - metadataKeys: Object.keys(metadata), - fromRunTags: event.runTags?.length ?? 0, - fromTelemetry: metadataKeyCount - (event.runTags?.filter((t) => t.includes(":")).length ?? 0), - }); - } - // #endregion @crumbs - // Set _llmUsage side-channel for dual-write to llm_usage_v1 const llmUsage: LlmUsageData = { genAiSystem: (props["gen_ai.system"] as string) ?? "unknown", diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts index 506a9360bd2..5b340eb4034 100644 --- a/internal-packages/llm-pricing/src/registry.ts +++ b/internal-packages/llm-pricing/src/registry.ts @@ -1,5 +1,4 @@ import type { PrismaClient } from "@trigger.dev/database"; -import { trail } from "agentcrumbs"; // @crumbs import type { LlmModelWithPricing, LlmCostResult, @@ -7,8 +6,6 @@ import type { PricingCondition, } from "./types.js"; -const crumb = trail("webapp:llm-pricing"); // @crumbs - type CompiledPattern = { regex: RegExp; model: LlmModelWithPricing; @@ -48,10 +45,7 @@ export class ModelPricingRegistry { orderBy: [{ startDate: "desc" }], }); - crumb("loaded models from db", { count: models.length }); // @crumbs - const compiled: CompiledPattern[] = []; - let skippedCount = 0; // @crumbs for (const model of models) { try { @@ -80,17 +74,14 @@ export class ModelPricingRegistry { }, }); } catch { - skippedCount++; // @crumbs // Skip models with invalid regex patterns console.warn(`Invalid regex pattern for model ${model.modelName}: ${model.matchPattern}`); - crumb("invalid regex pattern", { modelName: model.modelName, pattern: model.matchPattern }); // @crumbs } } this._patterns = compiled; this._exactMatchCache.clear(); this._loaded = true; - crumb("registry loaded", { patterns: compiled.length, skipped: skippedCount }); // @crumbs } async reload(): Promise { @@ -108,14 +99,12 @@ export class ModelPricingRegistry { for (const { regex, model } of this._patterns) { if (regex.test(responseModel)) { this._exactMatchCache.set(responseModel, model); - crumb("model matched", { responseModel, matchedModel: model.modelName }); // @crumbs return model; } } // Cache miss this._exactMatchCache.set(responseModel, null); - crumb("model not matched", { responseModel }); // @crumbs return null; } From a5b84cfdf322147f8398eb1df3a2faedee1cfaf7 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 13 Mar 2026 09:05:45 +0000 Subject: [PATCH 05/30] A bunch of improvements around extracting AI data New model admin dashboard, test model strings, add and edit models, view missing models and easily add them. Also extract cost data from ai gateway provider response metadata, better enrichment. --- .../components/runs/v3/ai/AIModelSummary.tsx | 12 +- .../runs/v3/ai/extractAISpanData.ts | 44 +- .../webapp/app/components/runs/v3/ai/types.ts | 3 + apps/webapp/app/env.server.ts | 6 + .../routes/admin.api.v1.llm-models.missing.ts | 33 + .../app/routes/admin.llm-models.$modelId.tsx | 449 +++++ .../app/routes/admin.llm-models._index.tsx | 346 ++++ .../admin.llm-models.missing.$model.tsx | 467 +++++ .../admin.llm-models.missing._index.tsx | 158 ++ .../app/routes/admin.llm-models.new.tsx | 397 +++++ apps/webapp/app/routes/admin.tsx | 4 + .../services/admin/missingLlmModels.server.ts | 127 ++ .../app/services/clickhouseInstance.server.ts | 28 + .../v3/utils/enrichCreatableEvents.server.ts | 138 +- apps/webapp/package.json | 1 + apps/webapp/seed-ai-spans.mts | 1564 +++++++++++++++++ internal-packages/llm-pricing/src/registry.ts | 12 + 17 files changed, 3747 insertions(+), 42 deletions(-) create mode 100644 apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts create mode 100644 apps/webapp/app/routes/admin.llm-models.$modelId.tsx create mode 100644 apps/webapp/app/routes/admin.llm-models._index.tsx create mode 100644 apps/webapp/app/routes/admin.llm-models.missing.$model.tsx create mode 100644 apps/webapp/app/routes/admin.llm-models.missing._index.tsx create mode 100644 apps/webapp/app/routes/admin.llm-models.new.tsx create mode 100644 apps/webapp/app/services/admin/missingLlmModels.server.ts create mode 100644 apps/webapp/seed-ai-spans.mts diff --git a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx index 9e627c844d3..a33b90ef899 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx @@ -6,6 +6,9 @@ export function AITagsRow({ aiData }: { aiData: AISpanData }) {
{aiData.model} {aiData.provider !== "unknown" && {aiData.provider}} + {aiData.resolvedProvider && ( + via {aiData.resolvedProvider} + )} {aiData.finishReason && {aiData.finishReason}} {aiData.serviceTier && tier: {aiData.serviceTier}} {aiData.toolChoice && tools: {aiData.toolChoice}} @@ -38,7 +41,14 @@ export function AIStatsSummary({ aiData }: { aiData: AISpanData }) { {aiData.cachedTokens != null && aiData.cachedTokens > 0 && ( - + + )} + {aiData.cacheCreationTokens != null && aiData.cacheCreationTokens > 0 && ( + )} {aiData.reasoningTokens != null && aiData.reasoningTokens > 0 && ( v ?? process.env.CLICKHOUSE_URL), + EVENTS_CLICKHOUSE_URL: z .string() .optional() diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts new file mode 100644 index 00000000000..5ca7077e1cc --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts @@ -0,0 +1,33 @@ +import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime"; +import { prisma } from "~/db.server"; +import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server"; +import { getMissingLlmModels } from "~/services/admin/missingLlmModels.server"; + +async function requireAdmin(request: Request) { + const authResult = await authenticateApiRequestWithPersonalAccessToken(request); + if (!authResult) { + throw json({ error: "Invalid or Missing API key" }, { status: 401 }); + } + + const user = await prisma.user.findUnique({ where: { id: authResult.userId } }); + if (!user?.admin) { + throw json({ error: "You must be an admin to perform this action" }, { status: 403 }); + } + + return user; +} + +export async function loader({ request }: LoaderFunctionArgs) { + await requireAdmin(request); + + const url = new URL(request.url); + const lookbackHours = parseInt(url.searchParams.get("lookbackHours") ?? "24", 10); + + if (isNaN(lookbackHours) || lookbackHours < 1 || lookbackHours > 720) { + return json({ error: "lookbackHours must be between 1 and 720" }, { status: 400 }); + } + + const models = await getMissingLlmModels({ lookbackHours }); + + return json({ models, lookbackHours }); +} diff --git a/apps/webapp/app/routes/admin.llm-models.$modelId.tsx b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx new file mode 100644 index 00000000000..4e72731cc3e --- /dev/null +++ b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx @@ -0,0 +1,449 @@ +import { Form, useNavigate } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "@remix-run/server-runtime"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { useState } from "react"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { Input } from "~/components/primitives/Input"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { prisma } from "~/db.server"; +import { requireUserId } from "~/services/session.server"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const model = await prisma.llmModel.findUnique({ + where: { friendlyId: params.modelId }, + include: { + pricingTiers: { include: { prices: true }, orderBy: { priority: "asc" } }, + }, + }); + + if (!model) throw new Response("Model not found", { status: 404 }); + + // Convert Prisma Decimal to plain numbers for serialization + const serialized = { + ...model, + pricingTiers: model.pricingTiers.map((t) => ({ + ...t, + prices: t.prices.map((p) => ({ ...p, price: Number(p.price) })), + })), + }; + + return typedjson({ model: serialized }); +}; + +const SaveSchema = z.object({ + modelName: z.string().min(1), + matchPattern: z.string().min(1), + pricingTiersJson: z.string(), +}); + +export async function action({ request, params }: ActionFunctionArgs) { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const friendlyId = params.modelId!; + const existing = await prisma.llmModel.findUnique({ where: { friendlyId } }); + if (!existing) throw new Response("Model not found", { status: 404 }); + const modelId = existing.id; + + const formData = await request.formData(); + const _action = formData.get("_action"); + + if (_action === "delete") { + await prisma.llmModel.delete({ where: { id: modelId } }); + await llmPricingRegistry?.reload(); + return redirect("/admin/llm-models"); + } + + if (_action === "save") { + const raw = Object.fromEntries(formData); + const parsed = SaveSchema.safeParse(raw); + + if (!parsed.success) { + return typedjson({ error: "Invalid form data", details: parsed.error.issues }, { status: 400 }); + } + + const { modelName, matchPattern, pricingTiersJson } = parsed.data; + + // Validate regex — strip (?i) POSIX flag since our registry handles it + try { + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); + } catch { + return typedjson({ error: "Invalid regex in matchPattern" }, { status: 400 }); + } + + // Parse tiers + let pricingTiers: Array<{ + name: string; + isDefault: boolean; + priority: number; + conditions: Array<{ usageDetailPattern: string; operator: string; value: number }>; + prices: Record; + }>; + try { + pricingTiers = JSON.parse(pricingTiersJson); + } catch { + return typedjson({ error: "Invalid pricing tiers JSON" }, { status: 400 }); + } + + // Update model + await prisma.llmModel.update({ + where: { id: modelId }, + data: { modelName, matchPattern }, + }); + + // Replace tiers + await prisma.llmPricingTier.deleteMany({ where: { modelId } }); + for (const tier of pricingTiers) { + await prisma.llmPricingTier.create({ + data: { + modelId, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId, + usageType, + price, + })), + }, + }, + }); + } + + await llmPricingRegistry?.reload(); + return typedjson({ success: true }); + } + + return typedjson({ error: "Unknown action" }, { status: 400 }); +} + +export default function AdminLlmModelDetailRoute() { + const { model } = useTypedLoaderData(); + const navigate = useNavigate(); + + const [modelName, setModelName] = useState(model.modelName); + const [matchPattern, setMatchPattern] = useState(model.matchPattern); + const [testInput, setTestInput] = useState(""); + const [tiers, setTiers] = useState(() => + model.pricingTiers.map((t) => ({ + name: t.name, + isDefault: t.isDefault, + priority: t.priority, + conditions: (t.conditions ?? []) as Array<{ + usageDetailPattern: string; + operator: string; + value: number; + }>, + prices: Object.fromEntries(t.prices.map((p) => [p.usageType, p.price])), + })) + ); + + // Test regex match + let testResult: boolean | null = null; + if (testInput) { + try { + const pattern = matchPattern.startsWith("(?i)") + ? matchPattern.slice(4) + : matchPattern; + testResult = new RegExp(pattern, "i").test(testInput); + } catch { + testResult = null; + } + } + + return ( +
+
+
+

{model.modelName}

+
+ + {model.source ?? "default"} + + + Back to list + +
+
+ +
+ + + +
+ {/* Model fields */} +
+ + setModelName(e.target.value)} + variant="medium" + fullWidth + /> +
+ +
+ + setMatchPattern(e.target.value)} + variant="medium" + fullWidth + className="font-mono text-xs" + /> +
+ + {/* Test pattern */} +
+ +
+ setTestInput(e.target.value)} + placeholder="Type a model name to test..." + variant="medium" + fullWidth + /> + {testInput && ( + + {testResult ? "Match" : "No match"} + + )} +
+
+ + {/* Pricing tiers */} +
+
+ + +
+ + {tiers.map((tier, tierIdx) => ( + { + const next = [...tiers]; + next[tierIdx] = updated; + setTiers(next); + }} + onRemove={() => setTiers(tiers.filter((_, i) => i !== tierIdx))} + /> + ))} +
+ + {/* Actions */} +
+ + + Cancel + +
+
+
+ + {/* Delete section */} +
+
{ + if (!confirm(`Delete model "${model.modelName}"?`)) e.preventDefault(); + }}> + + +
+
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Tier editor sub-component +// --------------------------------------------------------------------------- + +type TierData = { + name: string; + isDefault: boolean; + priority: number; + conditions: Array<{ usageDetailPattern: string; operator: string; value: number }>; + prices: Record; +}; + +const COMMON_USAGE_TYPES = [ + "input", + "output", + "input_cached_tokens", + "cache_creation_input_tokens", + "reasoning_tokens", +]; + +function TierEditor({ + tier, + onChange, + onRemove, +}: { + tier: TierData; + onChange: (t: TierData) => void; + onRemove: () => void; +}) { + const [newUsageType, setNewUsageType] = useState(""); + + return ( +
+
+
+ onChange({ ...tier, name: e.target.value })} + placeholder="Tier name" + /> + + +
+ +
+ + {/* Prices */} +
+ + Prices (per token) + +
+ {Object.entries(tier.prices).map(([usageType, price]) => ( +
+ {usageType} + { + const val = parseFloat(e.target.value); + if (!isNaN(val)) { + onChange({ + ...tier, + prices: { ...tier.prices, [usageType]: val }, + }); + } + }} + /> + +
+ ))} +
+ + {/* Add price */} +
+ + {newUsageType && ( + + )} +
+
+
+ ); +} diff --git a/apps/webapp/app/routes/admin.llm-models._index.tsx b/apps/webapp/app/routes/admin.llm-models._index.tsx new file mode 100644 index 00000000000..2bfa46d0c8a --- /dev/null +++ b/apps/webapp/app/routes/admin.llm-models._index.tsx @@ -0,0 +1,346 @@ +import { MagnifyingGlassIcon } from "@heroicons/react/20/solid"; +import { Form, useFetcher, Link } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "@remix-run/server-runtime"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { Input } from "~/components/primitives/Input"; +import { PaginationControls } from "~/components/primitives/Pagination"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { + Table, + TableBlankRow, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, +} from "~/components/primitives/Table"; +import { prisma } from "~/db.server"; +import { requireUserId } from "~/services/session.server"; +import { createSearchParams } from "~/utils/searchParams"; +import { seedLlmPricing } from "@internal/llm-pricing"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +const PAGE_SIZE = 50; + +const SearchParams = z.object({ + page: z.coerce.number().optional(), + search: z.string().optional(), +}); + +export const loader = async ({ request }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const searchParams = createSearchParams(request.url, SearchParams); + if (!searchParams.success) throw new Error(searchParams.error); + const { page: rawPage, search } = searchParams.params.getAll(); + const page = rawPage ?? 1; + + const where = { + projectId: null as string | null, + ...(search ? { modelName: { contains: search, mode: "insensitive" as const } } : {}), + }; + + const [rawModels, total] = await Promise.all([ + prisma.llmModel.findMany({ + where, + include: { + pricingTiers: { include: { prices: true }, orderBy: { priority: "asc" } }, + }, + orderBy: { modelName: "asc" }, + skip: (page - 1) * PAGE_SIZE, + take: PAGE_SIZE, + }), + prisma.llmModel.count({ where }), + ]); + + // Convert Prisma Decimal to plain numbers for serialization + const models = rawModels.map((m) => ({ + ...m, + pricingTiers: m.pricingTiers.map((t) => ({ + ...t, + prices: t.prices.map((p) => ({ ...p, price: Number(p.price) })), + })), + })); + + return typedjson({ + models, + total, + page, + pageCount: Math.ceil(total / PAGE_SIZE), + filters: { search }, + }); +}; + +export async function action({ request }: ActionFunctionArgs) { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const formData = await request.formData(); + const _action = formData.get("_action"); + + if (_action === "seed") { + console.log("[admin] seed action started"); + const result = await seedLlmPricing(prisma); + console.log(`[admin] seed complete: ${result.modelsCreated} created, ${result.modelsSkipped} skipped`); + await llmPricingRegistry?.reload(); + console.log("[admin] registry reloaded after seed"); + return typedjson({ + success: true, + message: `Seeded: ${result.modelsCreated} created, ${result.modelsSkipped} skipped`, + }); + } + + if (_action === "reload") { + console.log("[admin] reload action started"); + await llmPricingRegistry?.reload(); + console.log("[admin] registry reloaded"); + return typedjson({ success: true, message: "Registry reloaded" }); + } + + if (_action === "test") { + const modelString = formData.get("modelString"); + if (typeof modelString !== "string" || !modelString) { + return typedjson({ testResult: null }); + } + + // Use the registry's match() which handles prefix stripping automatically + const matched = llmPricingRegistry?.match(modelString) ?? null; + + return typedjson({ + testResult: { + modelString, + match: matched + ? { friendlyId: matched.friendlyId, modelName: matched.modelName } + : null, + }, + }); + } + + if (_action === "delete") { + const modelId = formData.get("modelId"); + if (typeof modelId === "string") { + await prisma.llmModel.delete({ where: { id: modelId } }); + await llmPricingRegistry?.reload(); + } + return typedjson({ success: true }); + } + + return typedjson({ error: "Unknown action" }, { status: 400 }); +} + +export default function AdminLlmModelsRoute() { + const { models, filters, page, pageCount, total } = + useTypedLoaderData(); + const seedFetcher = useFetcher(); + const reloadFetcher = useFetcher(); + const testFetcher = useFetcher<{ + testResult?: { + modelString: string; + match: { friendlyId: string; modelName: string } | null; + } | null; + }>(); + + const testResult = testFetcher.data?.testResult; + + return ( +
+
+
+
+ + +
+ +
+ + + + + + + + + + + + Missing models + + + + Add model + +
+
+ + {/* Model tester */} +
+ + + + + + + {testResult !== undefined && testResult !== null && ( +
+ + Testing: {testResult.modelString} + + {testResult.match ? ( +
+ Match:{" "} + + {testResult.match.modelName} + +
+ ) : ( +
+ No match found — this model has no pricing data +
+ )} +
+ )} +
+ +
+ + {total} global models (page {page} of {pageCount}) + + +
+ + + + + Model Name + Source + Input $/tok + Output $/tok + Other prices + + + + {models.length === 0 ? ( + + No models found + + ) : ( + models.map((model) => { + // Get default tier prices + const defaultTier = + model.pricingTiers.find((t) => t.isDefault) ?? model.pricingTiers[0]; + const priceMap = defaultTier + ? Object.fromEntries(defaultTier.prices.map((p) => [p.usageType, p.price])) + : {}; + const inputPrice = priceMap["input"]; + const outputPrice = priceMap["output"]; + const otherPrices = defaultTier + ? defaultTier.prices.filter( + (p) => p.usageType !== "input" && p.usageType !== "output" + ) + : []; + + return ( + + + + {model.modelName} + + + + + {model.source ?? "default"} + + + + + {inputPrice != null ? formatPrice(inputPrice) : "-"} + + + + + {outputPrice != null ? formatPrice(outputPrice) : "-"} + + + + {otherPrices.length > 0 ? ( + p.usageType).join(", ")}> + +{otherPrices.length} more + + ) : ( + - + )} + + + ); + }) + )} + +
+ + +
+
+ ); +} + +/** Format a per-token price as $/M tokens for readability */ +function formatPrice(perToken: number): string { + const perMillion = perToken * 1_000_000; + if (perMillion >= 1) return `$${perMillion.toFixed(2)}/M`; + if (perMillion >= 0.01) return `$${perMillion.toFixed(4)}/M`; + return `$${perMillion.toFixed(6)}/M`; +} diff --git a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx new file mode 100644 index 00000000000..7b6e83bf939 --- /dev/null +++ b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx @@ -0,0 +1,467 @@ +import { useState } from "react"; +import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "@remix-run/server-runtime"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { prisma } from "~/db.server"; +import { requireUserId } from "~/services/session.server"; +import { + getMissingModelSamples, + type MissingModelSample, +} from "~/services/admin/missingLlmModels.server"; + +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + // Model name is base64url-encoded in the URL param + const modelName = decodeURIComponent(params.model ?? ""); + if (!modelName) throw new Response("Missing model param", { status: 400 }); + + const url = new URL(request.url); + const lookbackHours = parseInt(url.searchParams.get("lookbackHours") ?? "24", 10); + + let samples: MissingModelSample[] = []; + let error: string | undefined; + + try { + samples = await getMissingModelSamples({ model: modelName, lookbackHours, limit: 10 }); + } catch (e) { + error = e instanceof Error ? e.message : "Failed to query ClickHouse"; + } + + return typedjson({ modelName, samples, lookbackHours, error }); +}; + +export default function AdminMissingModelDetailRoute() { + const { modelName, samples, lookbackHours, error } = useTypedLoaderData(); + const [copied, setCopied] = useState(false); + const [expandedSpans, setExpandedSpans] = useState>(new Set()); + + const providerCosts = extractProviderCosts(samples); + const prompt = buildPrompt(modelName, samples, providerCosts); + + function handleCopy() { + navigator.clipboard.writeText(prompt).then(() => { + setCopied(true); + setTimeout(() => setCopied(false), 2000); + }); + } + + function toggleSpan(spanId: string) { + setExpandedSpans((prev) => { + const next = new Set(prev); + if (next.has(spanId)) next.delete(spanId); + else next.add(spanId); + return next; + }); + } + + // Extract key token fields from the first sample for quick summary + const tokenSummary = samples.length > 0 ? extractTokenTypes(samples) : []; + + return ( +
+
+ {/* Header */} +
+
+

{modelName}

+ + Missing pricing — {samples.length} sample span{samples.length !== 1 ? "s" : ""} from + last {lookbackHours}h + +
+
+ + Add pricing + + + Back to missing + +
+
+ + {error && ( +
+ {error} +
+ )} + + {/* Token types summary */} + {tokenSummary.length > 0 && ( +
+ + Token types seen across samples + +
+ {tokenSummary.map((t) => ( + + {t.key} + + {t.min === t.max ? t.min.toLocaleString() : `${t.min.toLocaleString()}-${t.max.toLocaleString()}`} + + + ))} +
+ + These are the token usage types that need pricing entries (at minimum: input, output). + +
+ )} + + {/* Provider-reported costs */} + {providerCosts.length > 0 && ( +
+ + Provider-reported cost data found in {providerCosts.length} span{providerCosts.length !== 1 ? "s" : ""} + +
+ {providerCosts.map((c, i) => ( +
+ {c.source} + ${c.cost.toFixed(6)} + + ({c.inputTokens.toLocaleString()} in + {c.outputTokens.toLocaleString()} out) + +
+ ))} +
+ {providerCosts[0]?.estimatedInputPrice != null && ( +
+ + Estimated per-token rates (assuming ~3x output/input ratio): + +
+ input: {providerCosts[0].estimatedInputPrice.toExponential(4)} + output: {providerCosts[0].estimatedOutputPrice!.toExponential(4)} +
+ + Cross-reference with the provider's pricing page before using these estimates. + +
+ )} +
+ )} + + {/* Prompt section */} +
+
+ + Claude Code prompt — paste this to have it add pricing for this model + + +
+ 
+            {prompt}
+
+
+ + {/* Sample spans */} +
+ + Sample spans ({samples.length}) + + {samples.map((s) => { + const expanded = expandedSpans.has(s.span_id); + let parsedAttrs: Record | null = null; + try { + parsedAttrs = JSON.parse(s.attributes_text); + } catch { + // ignore + } + + return ( +
+ + {expanded && parsedAttrs && ( +
+ 
+                      {JSON.stringify(parsedAttrs, null, 2)}
+
+
+ )} +
+ ); + })} +
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Extract unique token usage types across all samples +// --------------------------------------------------------------------------- + +type TokenTypeSummary = { key: string; min: number; max: number }; + +function extractTokenTypes(samples: MissingModelSample[]): TokenTypeSummary[] { + const stats = new Map(); + + for (const s of samples) { + let attrs: Record; + try { + attrs = JSON.parse(s.attributes_text); + } catch { + continue; + } + + // Collect from gen_ai.usage.* + const genAiUsage = getNestedObj(attrs, ["gen_ai", "usage"]); + if (genAiUsage) { + for (const [k, v] of Object.entries(genAiUsage)) { + if (typeof v === "number" && v > 0) { + const existing = stats.get(`gen_ai.usage.${k}`); + if (existing) { + existing.min = Math.min(existing.min, v); + existing.max = Math.max(existing.max, v); + } else { + stats.set(`gen_ai.usage.${k}`, { min: v, max: v }); + } + } + } + } + + // Collect from ai.usage.* + const aiUsage = getNestedObj(attrs, ["ai", "usage"]); + if (aiUsage) { + for (const [k, v] of Object.entries(aiUsage)) { + if (typeof v === "number" && v > 0) { + const existing = stats.get(`ai.usage.${k}`); + if (existing) { + existing.min = Math.min(existing.min, v); + existing.max = Math.max(existing.max, v); + } else { + stats.set(`ai.usage.${k}`, { min: v, max: v }); + } + } + } + } + } + + return Array.from(stats.entries()) + .map(([key, { min, max }]) => ({ key, min, max })) + .sort((a, b) => a.key.localeCompare(b.key)); +} + +function getNestedObj( + obj: Record, + path: string[] +): Record | null { + let current: unknown = obj; + for (const key of path) { + if (!current || typeof current !== "object") return null; + current = (current as Record)[key]; + } + return current && typeof current === "object" ? (current as Record) : null; +} + +// --------------------------------------------------------------------------- +// Extract provider-reported costs from providerMetadata +// --------------------------------------------------------------------------- + +type ProviderCostInfo = { + source: string; // "gateway" or "openrouter" + cost: number; + inputTokens: number; + outputTokens: number; + estimatedInputPrice?: number; // per-token estimate + estimatedOutputPrice?: number; // per-token estimate +}; + +function extractProviderCosts(samples: MissingModelSample[]): ProviderCostInfo[] { + const costs: ProviderCostInfo[] = []; + + for (const s of samples) { + let attrs: Record; + try { + attrs = JSON.parse(s.attributes_text); + } catch { + continue; + } + + // Parse providerMetadata — could be nested or stringified + let providerMeta: Record | null = null; + const aiResponse = getNestedObj(attrs, ["ai", "response"]); + const rawMeta = aiResponse?.providerMetadata; + if (typeof rawMeta === "string") { + try { providerMeta = JSON.parse(rawMeta); } catch {} + } else if (rawMeta && typeof rawMeta === "object") { + providerMeta = rawMeta as Record; + } + if (!providerMeta) continue; + + // Get token counts + const genAiUsage = getNestedObj(attrs, ["gen_ai", "usage"]); + const inputTokens = Number(genAiUsage?.input_tokens ?? 0); + const outputTokens = Number(genAiUsage?.output_tokens ?? 0); + if (inputTokens === 0 && outputTokens === 0) continue; + + // Gateway: { gateway: { cost: "0.0006615" } } + const gw = getNestedObj(providerMeta, ["gateway"]); + if (gw) { + const cost = parseFloat(String(gw.cost ?? "0")); + if (cost > 0) { + costs.push({ source: "gateway", cost, inputTokens, outputTokens }); + continue; + } + } + + // OpenRouter: { openrouter: { usage: { cost: 0.000135 } } } + const or = getNestedObj(providerMeta, ["openrouter"]); + const orUsage = or ? getNestedObj(or, ["usage"]) : null; + if (orUsage) { + const cost = Number(orUsage.cost ?? 0); + if (cost > 0) { + costs.push({ source: "openrouter", cost, inputTokens, outputTokens }); + continue; + } + } + } + + // Estimate per-token prices from aggregate costs if we have enough data + if (costs.length > 0) { + // Use least-squares to estimate input/output price from cost = input*pi + output*po + // With 2+ samples we can solve; with 1 we can only estimate a blended rate + const totalInput = costs.reduce((s, c) => s + c.inputTokens, 0); + const totalOutput = costs.reduce((s, c) => s + c.outputTokens, 0); + const totalCost = costs.reduce((s, c) => s + c.cost, 0); + + if (totalInput > 0 && totalOutput > 0) { + // Simple approach: assume output is 2-5x input price (common ratio) + // Use ratio r where output_price = r * input_price + // totalCost = input_price * (totalInput + r * totalOutput) + // Try r=3 (common for many models) + const r = 3; + const estimatedInputPrice = totalCost / (totalInput + r * totalOutput); + const estimatedOutputPrice = estimatedInputPrice * r; + + for (const c of costs) { + c.estimatedInputPrice = estimatedInputPrice; + c.estimatedOutputPrice = estimatedOutputPrice; + } + } + } + + return costs; +} + +// --------------------------------------------------------------------------- +// Prompt builder — focused on figuring out pricing, not API mechanics +// --------------------------------------------------------------------------- + +function buildPrompt(modelName: string, samples: MissingModelSample[], providerCosts: ProviderCostInfo[]): string { + const hasPrefix = modelName.includes("/"); + const prefix = hasPrefix ? modelName.split("/")[0] : null; + const baseName = hasPrefix ? modelName.split("/").slice(1).join("/") : modelName; + + // Extract token types from samples + const tokenTypes = extractTokenTypes(samples); + const tokenTypeList = tokenTypes.length > 0 + ? tokenTypes.map((t) => ` - ${t.key}: ${t.min === t.max ? t.min : `${t.min}-${t.max}`}`).join("\n") + : " (no token data found in samples)"; + + // Get a compact sample of attributes for context + let sampleAttrs = ""; + if (samples.length > 0) { + try { + const attrs = JSON.parse(samples[0].attributes_text); + // Extract just the relevant fields + const compact: Record = {}; + if (attrs.gen_ai) compact.gen_ai = attrs.gen_ai; + if (attrs.ai?.usage) compact["ai.usage"] = attrs.ai.usage; + if (attrs.ai?.response?.providerMetadata) { + compact["ai.response.providerMetadata"] = attrs.ai.response.providerMetadata; + } + sampleAttrs = JSON.stringify(compact, null, 2); + } catch { + // ignore + } + } + + // Build suggested regex + const escapedBase = baseName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const suggestedPattern = prefix + ? `(?i)^(${prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/)?(${escapedBase})$` + : `(?i)^(${escapedBase})$`; + + return `I need to add LLM pricing for the model "${modelName}". + +## Model info +- Full model string from spans: \`${modelName}\` +- Base model name: \`${baseName}\`${prefix ? `\n- Provider prefix: \`${prefix}\`` : ""} +- This model appears in production spans but has no pricing data. + +## Token types seen in spans +${tokenTypeList} + +## What I need you to do + +1. **Look up pricing**: Find the current per-token pricing for \`${baseName}\` from the provider's official pricing page. Search the web if needed. + +2. **Present the pricing to me** in the following format so I can review before adding: + +\`\`\` +Model name: ${baseName} +Match pattern: ${suggestedPattern} +Pricing tier: Standard + +Prices (per token): + input: + output: + (add any additional token types if applicable) +\`\`\` + +**IMPORTANT: Do NOT call the admin API or create the model yourself.** Just research the pricing and present it to me. I will add it via the admin dashboard or ask you to proceed once I've reviewed. + +## Pricing research notes + +- All prices should be in **cost per token** (NOT per million). To convert: divide $/M by 1,000,000. + - Example: $3.00/M tokens = 0.000003 per token +- The \`matchPattern\` regex should match the model name both with and without the provider prefix. + - Suggested: \`${suggestedPattern}\` + - This matches both \`${baseName}\` and \`${modelName}\` +- Based on the token types seen in spans, check if the provider charges differently for: + - \`input\` and \`output\` — always required + - \`input_cached_tokens\` — if the provider offers prompt caching discounts + - \`cache_creation_input_tokens\` — if there's a cache write cost + - \`reasoning_tokens\` — if the model has chain-of-thought/reasoning tokens${providerCosts.length > 0 ? ` + +## Provider-reported costs (from ${providerCosts[0].source}) +The gateway/router is reporting costs for this model. Use these to cross-reference your pricing: +${providerCosts.map((c) => `- $${c.cost.toFixed(6)} for ${c.inputTokens.toLocaleString()} input + ${c.outputTokens.toLocaleString()} output tokens`).join("\n")}${providerCosts[0].estimatedInputPrice != null ? ` +- Estimated per-token rates (rough, assuming ~3x output/input ratio): + - input: ${providerCosts[0].estimatedInputPrice.toExponential(4)} (${(providerCosts[0].estimatedInputPrice * 1_000_000).toFixed(4)} $/M) + - output: ${providerCosts[0].estimatedOutputPrice!.toExponential(4)} (${(providerCosts[0].estimatedOutputPrice! * 1_000_000).toFixed(4)} $/M) +- Verify these against the official pricing page before using.` : ""}` : ""}${sampleAttrs ? ` + +## Sample span attributes (first span) +\`\`\`json +${sampleAttrs} +\`\`\`` : ""}`; +} diff --git a/apps/webapp/app/routes/admin.llm-models.missing._index.tsx b/apps/webapp/app/routes/admin.llm-models.missing._index.tsx new file mode 100644 index 00000000000..b778b9a3dc9 --- /dev/null +++ b/apps/webapp/app/routes/admin.llm-models.missing._index.tsx @@ -0,0 +1,158 @@ +import { useSearchParams } from "@remix-run/react"; +import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "@remix-run/server-runtime"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { LinkButton } from "~/components/primitives/Buttons"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { + Table, + TableBlankRow, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, +} from "~/components/primitives/Table"; +import { prisma } from "~/db.server"; +import { requireUserId } from "~/services/session.server"; +import { getMissingLlmModels } from "~/services/admin/missingLlmModels.server"; + +const LOOKBACK_OPTIONS = [ + { label: "1 hour", value: 1 }, + { label: "6 hours", value: 6 }, + { label: "24 hours", value: 24 }, + { label: "7 days", value: 168 }, + { label: "30 days", value: 720 }, +]; + +const SearchParams = z.object({ + lookbackHours: z.coerce.number().optional(), +}); + +export const loader = async ({ request }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const url = new URL(request.url); + const lookbackHours = parseInt(url.searchParams.get("lookbackHours") ?? "24", 10); + + let models: Awaited> = []; + let error: string | undefined; + + try { + models = await getMissingLlmModels({ lookbackHours }); + } catch (e) { + error = e instanceof Error ? e.message : "Failed to query ClickHouse"; + } + + return typedjson({ models, lookbackHours, error }); +}; + +export default function AdminLlmModelsMissingRoute() { + const { models, lookbackHours, error } = useTypedLoaderData(); + const [searchParams, setSearchParams] = useSearchParams(); + + return ( +
+
+
+

Missing LLM Models

+ + Back to models + +
+ + + Models appearing in spans without cost enrichment. These models need pricing data added. + + + {/* Lookback selector */} +
+ Lookback: + {LOOKBACK_OPTIONS.map((opt) => ( + + {opt.label} + + ))} +
+ + {error && ( +
+ {error} +
+ )} + + + {models.length} unpriced model{models.length !== 1 ? "s" : ""} found in the last{" "} + {lookbackHours < 24 + ? `${lookbackHours}h` + : lookbackHours < 168 + ? `${lookbackHours / 24}d` + : `${Math.round(lookbackHours / 24)}d`} + + + + + + Model Name + Provider + Span Count + Actions + + + + {models.length === 0 ? ( + + All models have pricing data + + ) : ( + models.map((m) => ( + + )) + )} + +
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Row component with link to detail page +// --------------------------------------------------------------------------- + +function MissingModelRow({ model: m }: { model: { model: string; system: string; count: number } }) { + return ( + + + + {m.model} + + + + {m.system || "-"} + + + {m.count.toLocaleString()} + + + + Details + + + + ); +} diff --git a/apps/webapp/app/routes/admin.llm-models.new.tsx b/apps/webapp/app/routes/admin.llm-models.new.tsx new file mode 100644 index 00000000000..b4cd957b740 --- /dev/null +++ b/apps/webapp/app/routes/admin.llm-models.new.tsx @@ -0,0 +1,397 @@ +import { Form, useActionData, useSearchParams } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "@remix-run/server-runtime"; +import { typedjson } from "remix-typedjson"; +import { z } from "zod"; +import { useState } from "react"; +import { Button, LinkButton } from "~/components/primitives/Buttons"; +import { Input } from "~/components/primitives/Input"; +import { prisma } from "~/db.server"; +import { requireUserId } from "~/services/session.server"; +import { generateFriendlyId } from "~/v3/friendlyIdentifiers"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +export const loader = async ({ request }: LoaderFunctionArgs) => { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + return typedjson({}); +}; + +const CreateSchema = z.object({ + modelName: z.string().min(1), + matchPattern: z.string().min(1), + pricingTiersJson: z.string(), +}); + +export async function action({ request }: ActionFunctionArgs) { + const userId = await requireUserId(request); + const user = await prisma.user.findUnique({ where: { id: userId } }); + if (!user?.admin) return redirect("/"); + + const formData = await request.formData(); + const raw = Object.fromEntries(formData); + console.log("[admin] create model form data:", JSON.stringify(raw).slice(0, 500)); + const parsed = CreateSchema.safeParse(raw); + + if (!parsed.success) { + console.log("[admin] create model validation error:", JSON.stringify(parsed.error.issues)); + return typedjson({ error: "Invalid form data", details: parsed.error.issues }, { status: 400 }); + } + + const { modelName, matchPattern, pricingTiersJson } = parsed.data; + + // Validate regex — strip (?i) POSIX flag since our registry handles it + try { + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); + } catch { + return typedjson({ error: "Invalid regex in matchPattern" }, { status: 400 }); + } + + let pricingTiers: Array<{ + name: string; + isDefault: boolean; + priority: number; + conditions: Array<{ usageDetailPattern: string; operator: string; value: number }>; + prices: Record; + }>; + try { + pricingTiers = JSON.parse(pricingTiersJson); + } catch { + return typedjson({ error: "Invalid pricing tiers JSON" }, { status: 400 }); + } + + const model = await prisma.llmModel.create({ + data: { + friendlyId: generateFriendlyId("llm_model"), + modelName, + matchPattern, + source: "admin", + }, + }); + + for (const tier of pricingTiers) { + await prisma.llmPricingTier.create({ + data: { + modelId: model.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId: model.id, + usageType, + price, + })), + }, + }, + }); + } + + await llmPricingRegistry?.reload(); + return redirect(`/admin/llm-models/${model.friendlyId}`); +} + +export default function AdminLlmModelNewRoute() { + const actionData = useActionData<{ error?: string; details?: unknown[] }>(); + const [params] = useSearchParams(); + const initialModelName = params.get("modelName") ?? ""; + const [modelName, setModelName] = useState(initialModelName); + const [matchPattern, setMatchPattern] = useState(""); + const [testInput, setTestInput] = useState(""); + const [tiers, setTiers] = useState([ + { name: "Standard", isDefault: true, priority: 0, conditions: [], prices: { input: 0, output: 0 } }, + ]); + + let testResult: boolean | null = null; + if (testInput && matchPattern) { + try { + const pattern = matchPattern.startsWith("(?i)") + ? matchPattern.slice(4) + : matchPattern; + testResult = new RegExp(pattern, "i").test(testInput); + } catch { + testResult = null; + } + } + + // Auto-generate match pattern from model name + function autoPattern() { + if (modelName) { + const escaped = modelName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + setMatchPattern(`(?i)^(${escaped})$`); + } + } + + return ( +
+
+
+

New LLM Model

+ + Back to list + +
+ +
+ + +
+
+ + setModelName(e.target.value)} + variant="medium" + fullWidth + placeholder="e.g. gemini-3-flash" + /> +
+ +
+
+ + +
+ setMatchPattern(e.target.value)} + variant="medium" + fullWidth + className="font-mono text-xs" + placeholder="(?i)^(google/)?(gemini-3-flash)$" + /> +
+ +
+ +
+ setTestInput(e.target.value)} + placeholder="Type a model name to test..." + variant="medium" + fullWidth + /> + {testInput && ( + + {testResult ? "Match" : "No match"} + + )} +
+
+ + {/* Pricing tiers */} +
+
+ + +
+ + {tiers.map((tier, tierIdx) => ( + { + const next = [...tiers]; + next[tierIdx] = updated; + setTiers(next); + }} + onRemove={() => setTiers(tiers.filter((_, i) => i !== tierIdx))} + /> + ))} +
+ + {actionData?.error && ( +
+ {actionData.error} + {actionData.details && ( + 
+                    {JSON.stringify(actionData.details, null, 2)}
+
+ )} +
+ )} + +
+ + + Cancel + +
+
+
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Shared tier editor (duplicated from detail page — could be extracted later) +// --------------------------------------------------------------------------- + +type TierData = { + name: string; + isDefault: boolean; + priority: number; + conditions: Array<{ usageDetailPattern: string; operator: string; value: number }>; + prices: Record; +}; + +const COMMON_USAGE_TYPES = [ + "input", + "output", + "input_cached_tokens", + "cache_creation_input_tokens", + "reasoning_tokens", +]; + +function TierEditor({ + tier, + onChange, + onRemove, +}: { + tier: TierData; + onChange: (t: TierData) => void; + onRemove: () => void; +}) { + const [newUsageType, setNewUsageType] = useState(""); + + return ( +
+
+
+ onChange({ ...tier, name: e.target.value })} + placeholder="Tier name" + /> + + +
+ +
+ +
+ Prices (per token) +
+ {Object.entries(tier.prices).map(([usageType, price]) => ( +
+ {usageType} + { + const val = parseFloat(e.target.value); + if (!isNaN(val)) { + onChange({ ...tier, prices: { ...tier.prices, [usageType]: val } }); + } + }} + /> + +
+ ))} +
+ +
+ + {newUsageType && ( + + )} +
+
+
+ ); +} diff --git a/apps/webapp/app/routes/admin.tsx b/apps/webapp/app/routes/admin.tsx index ac8e56c855e..34792e66ee5 100644 --- a/apps/webapp/app/routes/admin.tsx +++ b/apps/webapp/app/routes/admin.tsx @@ -32,6 +32,10 @@ export default function Page() { label: "Concurrency", to: "/admin/concurrency", }, + { + label: "LLM Models", + to: "/admin/llm-models", + }, ]} layoutId={"admin"} /> diff --git a/apps/webapp/app/services/admin/missingLlmModels.server.ts b/apps/webapp/app/services/admin/missingLlmModels.server.ts new file mode 100644 index 00000000000..01237ef2b57 --- /dev/null +++ b/apps/webapp/app/services/admin/missingLlmModels.server.ts @@ -0,0 +1,127 @@ +import { adminClickhouseClient } from "~/services/clickhouseInstance.server"; +import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; + +export type MissingLlmModel = { + model: string; + system: string; + count: number; +}; + +export async function getMissingLlmModels(opts: { + lookbackHours?: number; +} = {}): Promise { + const lookbackHours = opts.lookbackHours ?? 24; + const since = new Date(Date.now() - lookbackHours * 60 * 60 * 1000); + + // queryBuilderFast returns a factory function — call it to get the builder + const createBuilder = adminClickhouseClient.reader.queryBuilderFast<{ + model: string; + system: string; + cnt: string; + }>({ + name: "missingLlmModels", + table: "trigger_dev.task_events_v2", + columns: [ + { name: "model", expression: "attributes.gen_ai.response.model.:String" }, + { name: "system", expression: "attributes.gen_ai.system.:String" }, + { name: "cnt", expression: "count()" }, + ], + }); + const qb = createBuilder(); + + // Partition pruning on inserted_at (partition key is toDate(inserted_at)) + qb.where("inserted_at >= {since: DateTime64(3)}", { + since: formatDateTime(since), + }); + + // Only spans that have a model set + qb.where("attributes.gen_ai.response.model.:String != {empty: String}", { empty: "" }); + + // Only spans that were NOT cost-enriched (trigger.llm.total_cost is NULL) + qb.where("attributes.trigger.llm.total_cost.:Float64 IS NULL", {}); + + // Only completed spans + qb.where("kind = {kind: String}", { kind: "SPAN" }); + qb.where("status = {status: String}", { status: "OK" }); + + qb.groupBy("model, system"); + qb.orderBy("cnt DESC"); + qb.limit(100); + + const [err, rows] = await qb.execute(); + + if (err) { + throw err; + } + + if (!rows) { + return []; + } + + const candidates = rows + .filter((r) => r.model) + .map((r) => ({ + model: r.model, + system: r.system, + count: parseInt(r.cnt, 10), + })); + + if (candidates.length === 0) return []; + + // Filter out models that now have pricing in the database (added after spans were inserted). + // The registry's match() handles prefix stripping for gateway/openrouter models. + return candidates.filter((c) => !llmPricingRegistry?.match(c.model)); +} + +export type MissingModelSample = { + span_id: string; + run_id: string; + message: string; + attributes_text: string; + duration: string; + start_time: string; +}; + +export async function getMissingModelSamples(opts: { + model: string; + lookbackHours?: number; + limit?: number; +}): Promise { + const lookbackHours = opts.lookbackHours ?? 24; + const limit = opts.limit ?? 10; + const since = new Date(Date.now() - lookbackHours * 60 * 60 * 1000); + + const createBuilder = adminClickhouseClient.reader.queryBuilderFast({ + name: "missingModelSamples", + table: "trigger_dev.task_events_v2", + columns: [ + "span_id", + "run_id", + "message", + "attributes_text", + "duration", + "start_time", + ], + }); + const qb = createBuilder(); + + qb.where("inserted_at >= {since: DateTime64(3)}", { since: formatDateTime(since) }); + qb.where("attributes.gen_ai.response.model.:String = {model: String}", { model: opts.model }); + qb.where("attributes.trigger.llm.total_cost.:Float64 IS NULL", {}); + qb.where("kind = {kind: String}", { kind: "SPAN" }); + qb.where("status = {status: String}", { status: "OK" }); + qb.orderBy("start_time DESC"); + qb.limit(limit); + + const [err, rows] = await qb.execute(); + + if (err) { + throw err; + } + + return rows ?? []; +} + +function formatDateTime(date: Date): string { + return date.toISOString().replace("T", " ").replace("Z", ""); +} diff --git a/apps/webapp/app/services/clickhouseInstance.server.ts b/apps/webapp/app/services/clickhouseInstance.server.ts index 61494811a0e..9c4941671f3 100644 --- a/apps/webapp/app/services/clickhouseInstance.server.ts +++ b/apps/webapp/app/services/clickhouseInstance.server.ts @@ -71,6 +71,34 @@ function initializeLogsClickhouseClient() { }); } +export const adminClickhouseClient = singleton( + "adminClickhouseClient", + initializeAdminClickhouseClient +); + +function initializeAdminClickhouseClient() { + if (!env.ADMIN_CLICKHOUSE_URL) { + throw new Error("ADMIN_CLICKHOUSE_URL is not set"); + } + + const url = new URL(env.ADMIN_CLICKHOUSE_URL); + url.searchParams.delete("secure"); + + return new ClickHouse({ + url: url.toString(), + name: "admin-clickhouse", + keepAlive: { + enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.CLICKHOUSE_LOG_LEVEL, + compression: { + request: true, + }, + maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }); +} + export const queryClickhouseClient = singleton( "queryClickhouseClient", initializeQueryClickhouseClient diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index 1c125060ae2..a062e491de3 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -76,42 +76,66 @@ function enrichLlmCost(event: CreateEventInput): void { return; } - if (!_registry?.isLoaded) { - return; - } - - const cost = _registry.calculateCost(responseModel, usageDetails); - if (!cost) return; - - // Add trigger.llm.* attributes to the span - event.properties = { - ...props, - "trigger.llm.input_cost": cost.inputCost, - "trigger.llm.output_cost": cost.outputCost, - "trigger.llm.total_cost": cost.totalCost, - "trigger.llm.matched_model": cost.matchedModelName, - "trigger.llm.matched_model_id": cost.matchedModelId, - "trigger.llm.pricing_tier": cost.pricingTierName, - "trigger.llm.pricing_tier_id": cost.pricingTierId, - }; - - // Add style accessories for model, tokens, and cost + // Add style accessories for model and tokens (even without cost data) const inputTokens = usageDetails["input"] ?? 0; const outputTokens = usageDetails["output"] ?? 0; const totalTokens = inputTokens + outputTokens; + const pillItems: Array<{ text: string; icon: string }> = [ + { text: responseModel, icon: "tabler-cube" }, + { text: formatTokenCount(totalTokens), icon: "tabler-hash" }, + ]; + + // Try cost enrichment if the registry is loaded. + // The registry handles prefix stripping (e.g. "mistral/mistral-large-3" → "mistral-large-3") + // for gateway/openrouter models automatically in its match() method. + let cost: ReturnType["calculateCost"]> | null = null; + if (_registry?.isLoaded) { + cost = _registry.calculateCost(responseModel, usageDetails); + } + + // Fallback: extract cost from provider metadata (gateway/openrouter report per-request cost) + let providerCost: { totalCost: number; source: string } | null = null; + if (!cost) { + providerCost = extractProviderCost(props); + } + + if (cost) { + // Add trigger.llm.* attributes to the span from our pricing registry + event.properties = { + ...props, + "trigger.llm.input_cost": cost.inputCost, + "trigger.llm.output_cost": cost.outputCost, + "trigger.llm.total_cost": cost.totalCost, + "trigger.llm.matched_model": cost.matchedModelName, + "trigger.llm.matched_model_id": cost.matchedModelId, + "trigger.llm.pricing_tier": cost.pricingTierName, + "trigger.llm.pricing_tier_id": cost.pricingTierId, + }; + + pillItems.push({ text: formatCost(cost.totalCost), icon: "tabler-currency-dollar" }); + } else if (providerCost) { + // Use provider-reported cost as fallback (no input/output breakdown available) + event.properties = { + ...props, + "trigger.llm.total_cost": providerCost.totalCost, + "trigger.llm.cost_source": providerCost.source, + }; + + pillItems.push({ text: formatCost(providerCost.totalCost), icon: "tabler-currency-dollar" }); + } + event.style = { ...event.style, accessory: { style: "pills", - items: [ - { text: responseModel, icon: "tabler-cube" }, - { text: formatTokenCount(totalTokens), icon: "tabler-hash" }, - { text: formatCost(cost.totalCost), icon: "tabler-currency-dollar" }, - ], + items: pillItems, }, }; + // Only write llm_usage when cost data is available + if (!cost && !providerCost) return; + // Build metadata map from run tags and ai.telemetry.metadata.* const metadata: Record = {}; @@ -135,18 +159,18 @@ function enrichLlmCost(event: CreateEventInput): void { genAiSystem: (props["gen_ai.system"] as string) ?? "unknown", requestModel: (props["gen_ai.request.model"] as string) ?? responseModel, responseModel, - matchedModelId: cost.matchedModelId, + matchedModelId: cost?.matchedModelId ?? "", operationName: (props["gen_ai.operation.name"] as string) ?? (props["operation.name"] as string) ?? "", - pricingTierId: cost.pricingTierId, - pricingTierName: cost.pricingTierName, + pricingTierId: cost?.pricingTierId ?? (providerCost ? `provider:${providerCost.source}` : ""), + pricingTierName: cost?.pricingTierName ?? (providerCost ? `${providerCost.source} reported` : ""), inputTokens: usageDetails["input"] ?? 0, outputTokens: usageDetails["output"] ?? 0, totalTokens: Object.values(usageDetails).reduce((sum, v) => sum + v, 0), usageDetails, - inputCost: cost.inputCost, - outputCost: cost.outputCost, - totalCost: cost.totalCost, - costDetails: cost.costDetails, + inputCost: cost?.inputCost ?? 0, + outputCost: cost?.outputCost ?? 0, + totalCost: cost?.totalCost ?? providerCost?.totalCost ?? 0, + costDetails: cost?.costDetails ?? {}, metadata, }; @@ -197,6 +221,15 @@ function enrichStyle(event: CreateEventInput) { // GenAI System check const system = props["gen_ai.system"]; if (typeof system === "string") { + // For gateway/openrouter, derive the icon from the model's provider prefix + // e.g. "mistral/mistral-large-3" → "mistral", "anthropic/claude-..." → "anthropic" + if (system === "gateway" || system === "openrouter") { + const modelId = props["gen_ai.request.model"] ?? props["ai.model.id"]; + if (typeof modelId === "string" && modelId.includes("/")) { + const provider = modelId.split("/")[0].replace(/-/g, ""); + return { ...baseStyle, icon: `tabler-brand-${provider}` }; + } + } return { ...baseStyle, icon: `tabler-brand-${system.split(".")[0]}` }; } @@ -225,6 +258,47 @@ function formatTokenCount(tokens: number): string { return tokens.toString(); } +/** + * Extract provider-reported cost from ai.response.providerMetadata. + * Gateway and OpenRouter include per-request cost in their metadata. + */ +function extractProviderCost( + props: Record +): { totalCost: number; source: string } | null { + const rawMeta = props["ai.response.providerMetadata"]; + if (typeof rawMeta !== "string") return null; + + let meta: Record; + try { + meta = JSON.parse(rawMeta); + } catch { + return null; + } + + if (!meta || typeof meta !== "object") return null; + + // Gateway: { gateway: { cost: "0.0006615" } } + const gateway = meta.gateway; + if (gateway && typeof gateway === "object") { + const gw = gateway as Record; + const cost = parseFloat(String(gw.cost ?? "0")); + if (cost > 0) return { totalCost: cost, source: "gateway" }; + } + + // OpenRouter: { openrouter: { usage: { cost: 0.000135 } } } + const openrouter = meta.openrouter; + if (openrouter && typeof openrouter === "object") { + const or = openrouter as Record; + const usage = or.usage; + if (usage && typeof usage === "object") { + const cost = Number((usage as Record).cost ?? 0); + if (cost > 0) return { totalCost: cost, source: "openrouter" }; + } + } + + return null; +} + function formatCost(cost: number): string { if (cost >= 1) return `$${cost.toFixed(2)}`; if (cost >= 0.01) return `$${cost.toFixed(4)}`; diff --git a/apps/webapp/package.json b/apps/webapp/package.json index fce7f2769c1..48994376066 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -16,6 +16,7 @@ "start:local": "cross-env node --max-old-space-size=8192 ./build/server.js", "typecheck": "cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsc --noEmit -p ./tsconfig.check.json", "db:seed": "tsx seed.mts", + "db:seed:ai-spans": "tsx seed-ai-spans.mts", "upload:sourcemaps": "bash ./upload-sourcemaps.sh", "test": "vitest --no-file-parallelism", "eval:dev": "evalite watch" diff --git a/apps/webapp/seed-ai-spans.mts b/apps/webapp/seed-ai-spans.mts new file mode 100644 index 00000000000..19a72c23f22 --- /dev/null +++ b/apps/webapp/seed-ai-spans.mts @@ -0,0 +1,1564 @@ +import { trail } from "agentcrumbs"; // @crumbs +const crumb = trail("webapp"); // @crumbs +import { prisma } from "./app/db.server"; +import { createOrganization } from "./app/models/organization.server"; +import { createProject } from "./app/models/project.server"; +import { ClickHouse } from "@internal/clickhouse"; +import type { TaskEventV2Input, LlmUsageV1Input } from "@internal/clickhouse"; +import { + generateTraceId, + generateSpanId, +} from "./app/v3/eventRepository/common.server"; +import { + enrichCreatableEvents, + setLlmPricingRegistry, +} from "./app/v3/utils/enrichCreatableEvents.server"; +import { ModelPricingRegistry, seedLlmPricing } from "@internal/llm-pricing"; +import { nanoid } from "nanoid"; +import { unflattenAttributes } from "@trigger.dev/core/v3/utils/flattenAttributes"; +import type { Attributes } from "@opentelemetry/api"; +import type { CreateEventInput } from "./app/v3/eventRepository/eventRepository.types"; + +const ORG_TITLE = "AI Spans Dev"; +const PROJECT_NAME = "ai-chat-demo"; +const TASK_SLUG = "ai-chat"; +const QUEUE_NAME = "task/ai-chat"; +const WORKER_VERSION = "seed-ai-spans-v1"; + +// --------------------------------------------------------------------------- +// ClickHouse formatting helpers (replicated from clickhouseEventRepository) +// --------------------------------------------------------------------------- + +function formatStartTime(startTimeNs: bigint): string { + const str = startTimeNs.toString(); + if (str.length !== 19) return str; + return str.substring(0, 10) + "." + str.substring(10); +} + +function formatDuration(value: number | bigint): string { + if (value < 0) return "0"; + if (typeof value === "bigint") return value.toString(); + return Math.floor(value).toString(); +} + +function formatClickhouseDateTime(date: Date): string { + return date.toISOString().replace("T", " ").replace("Z", ""); +} + +function removePrivateProperties(attributes: Attributes): Attributes | undefined { + const result: Attributes = {}; + for (const [key, value] of Object.entries(attributes)) { + if (key.startsWith("$") || key.startsWith("ctx.")) continue; + result[key] = value; + } + return Object.keys(result).length === 0 ? undefined : result; +} + +function eventToClickhouseRow(event: CreateEventInput): TaskEventV2Input { + // kind + let kind: string; + if (event.kind === "UNSPECIFIED") kind = "ANCESTOR_OVERRIDE"; + else if (event.level === "TRACE") kind = "SPAN"; + else if (event.isDebug) kind = "DEBUG_EVENT"; + else kind = `LOG_${(event.level ?? "LOG").toString().toUpperCase()}`; + + // status + let status: string; + if (event.isPartial) status = "PARTIAL"; + else if (event.isError) status = "ERROR"; + else if (event.isCancelled) status = "CANCELLED"; + else status = "OK"; + + // attributes + const publicAttrs = removePrivateProperties(event.properties as Attributes); + const unflattened = publicAttrs ? unflattenAttributes(publicAttrs) : {}; + const attributes = + unflattened && typeof unflattened === "object" ? { ...unflattened } : {}; + + // metadata — mirrors createEventToTaskEventV1InputMetadata + const metadataObj: Record = {}; + if (event.style) { + metadataObj.style = unflattenAttributes(event.style as Attributes); + } + if (event.attemptNumber) { + metadataObj.attemptNumber = event.attemptNumber; + } + // Extract entity from properties (SemanticInternalAttributes) + const entityType = event.properties?.["$entity.type"]; + if (typeof entityType === "string") { + metadataObj.entity = { + entityType, + entityId: event.properties?.["$entity.id"] as string | undefined, + entityMetadata: event.properties?.["$entity.metadata"] as string | undefined, + }; + } + const metadata = JSON.stringify(metadataObj); + + return { + environment_id: event.environmentId, + organization_id: event.organizationId, + project_id: event.projectId, + task_identifier: event.taskSlug, + run_id: event.runId, + start_time: formatStartTime(BigInt(event.startTime)), + duration: formatDuration(event.duration ?? 0), + trace_id: event.traceId, + span_id: event.spanId, + parent_span_id: event.parentId ?? "", + message: event.message, + kind, + status, + attributes, + metadata, + expires_at: formatClickhouseDateTime( + new Date(Date.now() + 365 * 24 * 60 * 60 * 1000) + ), + machine_id: "", + }; +} + +function eventToLlmUsageRow(event: CreateEventInput): LlmUsageV1Input { + const llm = event._llmUsage!; + return { + organization_id: event.organizationId, + project_id: event.projectId, + environment_id: event.environmentId, + run_id: event.runId, + task_identifier: event.taskSlug, + trace_id: event.traceId, + span_id: event.spanId, + gen_ai_system: llm.genAiSystem, + request_model: llm.requestModel, + response_model: llm.responseModel, + matched_model_id: llm.matchedModelId, + operation_name: llm.operationName, + pricing_tier_id: llm.pricingTierId, + pricing_tier_name: llm.pricingTierName, + input_tokens: llm.inputTokens, + output_tokens: llm.outputTokens, + total_tokens: llm.totalTokens, + usage_details: llm.usageDetails, + input_cost: llm.inputCost, + output_cost: llm.outputCost, + total_cost: llm.totalCost, + cost_details: llm.costDetails, + metadata: llm.metadata, + start_time: formatStartTime(BigInt(event.startTime)), + duration: formatDuration(event.duration ?? 0), + }; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function seedAiSpans() { + crumb("seed started"); // @crumbs + console.log("Starting AI span seed...\n"); + + // 1. Find user + crumb("finding user"); // @crumbs + const user = await prisma.user.findUnique({ + where: { email: "local@trigger.dev" }, + }); + if (!user) { + console.error("User local@trigger.dev not found. Run `pnpm run db:seed` first."); + process.exit(1); + } + crumb("user found", { userId: user.id }); // @crumbs + + // 2. Find or create org + crumb("finding/creating org"); // @crumbs + let org = await prisma.organization.findFirst({ + where: { title: ORG_TITLE, members: { some: { userId: user.id } } }, + }); + if (!org) { + org = await createOrganization({ title: ORG_TITLE, userId: user.id, companySize: "1-10" }); + console.log(`Created org: ${org.title} (${org.slug})`); + } else { + console.log(`Org exists: ${org.title} (${org.slug})`); + } + crumb("org ready", { orgId: org.id, slug: org.slug }); // @crumbs + + // 3. Find or create project + crumb("finding/creating project"); // @crumbs + let project = await prisma.project.findFirst({ + where: { name: PROJECT_NAME, organizationId: org.id }, + }); + if (!project) { + project = await createProject({ + organizationSlug: org.slug, + name: PROJECT_NAME, + userId: user.id, + version: "v3", + }); + console.log(`Created project: ${project.name} (${project.externalRef})`); + } else { + console.log(`Project exists: ${project.name} (${project.externalRef})`); + } + crumb("project ready", { projectId: project.id, ref: project.externalRef }); // @crumbs + + // 4. Get DEVELOPMENT environment + crumb("finding dev environment"); // @crumbs + const runtimeEnv = await prisma.runtimeEnvironment.findFirst({ + where: { projectId: project.id, type: "DEVELOPMENT" }, + }); + if (!runtimeEnv) { + console.error("No DEVELOPMENT environment found for project."); + process.exit(1); + } + crumb("dev env found", { envId: runtimeEnv.id }); // @crumbs + + // 5. Upsert background worker + crumb("upserting worker/task/queue"); // @crumbs + const worker = await prisma.backgroundWorker.upsert({ + where: { + projectId_runtimeEnvironmentId_version: { + projectId: project.id, + runtimeEnvironmentId: runtimeEnv.id, + version: WORKER_VERSION, + }, + }, + update: {}, + create: { + friendlyId: `worker_${nanoid()}`, + engine: "V2", + contentHash: `seed-ai-spans-${Date.now()}`, + sdkVersion: "3.0.0", + cliVersion: "3.0.0", + projectId: project.id, + runtimeEnvironmentId: runtimeEnv.id, + version: WORKER_VERSION, + metadata: {}, + }, + }); + + // 6. Upsert task + await prisma.backgroundWorkerTask.upsert({ + where: { workerId_slug: { workerId: worker.id, slug: TASK_SLUG } }, + update: {}, + create: { + friendlyId: `task_${nanoid()}`, + slug: TASK_SLUG, + filePath: "src/trigger/ai-chat.ts", + exportName: "aiChat", + workerId: worker.id, + projectId: project.id, + runtimeEnvironmentId: runtimeEnv.id, + }, + }); + + // 7. Upsert queue + await prisma.taskQueue.upsert({ + where: { + runtimeEnvironmentId_name: { + runtimeEnvironmentId: runtimeEnv.id, + name: QUEUE_NAME, + }, + }, + update: {}, + create: { + friendlyId: `queue_${nanoid()}`, + name: QUEUE_NAME, + projectId: project.id, + runtimeEnvironmentId: runtimeEnv.id, + }, + }); + + crumb("infra upserts done"); // @crumbs + + // 8. Create the TaskRun + crumb("creating TaskRun"); // @crumbs + const traceId = generateTraceId(); + const rootSpanId = generateSpanId(); + const now = Date.now(); + // Spans start at `now` and extend into the future. completedAt must cover + // the full span tree so getSpan's start_time <= completedAt filter works. + const startedAt = new Date(now); + const completedAt = new Date(now + 150_000); // 2.5 min to cover all spans + + const run = await prisma.taskRun.create({ + data: { + friendlyId: `run_${nanoid()}`, + engine: "V2", + status: "COMPLETED_SUCCESSFULLY", + taskIdentifier: TASK_SLUG, + payload: JSON.stringify({ + message: "What is the current Federal Reserve interest rate?", + }), + payloadType: "application/json", + traceId, + spanId: rootSpanId, + runtimeEnvironmentId: runtimeEnv.id, + projectId: project.id, + organizationId: org.id, + queue: QUEUE_NAME, + lockedToVersionId: worker.id, + startedAt, + completedAt, + runTags: ["user:seed_user_42", "chat:seed_session"], + taskEventStore: "clickhouse_v2", + }, + }); + + crumb("TaskRun created", { runId: run.friendlyId, traceId }); // @crumbs + console.log(`Created TaskRun: ${run.friendlyId}`); + + // 9. Build span tree + crumb("building span tree"); // @crumbs + const events = buildAiSpanTree({ + traceId, + rootSpanId, + runId: run.friendlyId, + environmentId: runtimeEnv.id, + projectId: project.id, + organizationId: org.id, + taskSlug: TASK_SLUG, + baseTimeMs: now, + }); + + crumb("span tree built", { spanCount: events.length }); // @crumbs + console.log(`Built ${events.length} spans`); + + // 10. Seed LLM pricing and enrich + crumb("seeding LLM pricing"); // @crumbs + const seedResult = await seedLlmPricing(prisma); + console.log( + `LLM pricing: ${seedResult.modelsCreated} created, ${seedResult.modelsSkipped} skipped` + ); + + crumb("loading pricing registry"); // @crumbs + const registry = new ModelPricingRegistry(prisma); + setLlmPricingRegistry(registry); + await registry.loadFromDatabase(); + + crumb("enriching events"); // @crumbs + const enriched = enrichCreatableEvents(events); + + const enrichedCount = enriched.filter((e) => e._llmUsage != null).length; + const totalCost = enriched.reduce((sum, e) => sum + (e._llmUsage?.totalCost ?? 0), 0); + console.log( + `Enriched ${enrichedCount} spans with LLM cost (total: $${totalCost.toFixed(6)})` + ); + + crumb("enrichment done", { enrichedCount, totalCost }); // @crumbs + + // 11. Insert into ClickHouse + crumb("inserting into ClickHouse"); // @crumbs + const clickhouseUrl = process.env.CLICKHOUSE_URL ?? process.env.EVENTS_CLICKHOUSE_URL; + if (!clickhouseUrl) { + console.error("CLICKHOUSE_URL or EVENTS_CLICKHOUSE_URL not set"); + process.exit(1); + } + + const url = new URL(clickhouseUrl); + url.searchParams.delete("secure"); + const clickhouse = new ClickHouse({ url: url.toString() }); + + // Convert to ClickHouse rows and insert + const chRows = enriched.map(eventToClickhouseRow); + await clickhouse.taskEventsV2.insert(chRows); + + crumb("task events inserted", { rowCount: chRows.length }); // @crumbs + + // Insert LLM usage rows + const llmRows = enriched.filter((e) => e._llmUsage != null).map(eventToLlmUsageRow); + if (llmRows.length > 0) { + await clickhouse.llmUsage.insert(llmRows); + crumb("llm usage inserted", { rowCount: llmRows.length }); // @crumbs + } + + // 12. Output + console.log("\nDone!\n"); + console.log( + `Run URL: http://localhost:3030/orgs/${org.slug}/projects/${project.slug}/env/dev/runs/${run.friendlyId}` + ); + console.log(`Spans: ${events.length}`); + console.log(`LLM cost enriched: ${enrichedCount}`); + console.log(`Total cost: $${totalCost.toFixed(6)}`); + crumb("seed complete"); // @crumbs + process.exit(0); +} + +// --------------------------------------------------------------------------- +// Span tree builder +// --------------------------------------------------------------------------- + +type SpanTreeParams = { + traceId: string; + rootSpanId: string; + runId: string; + environmentId: string; + projectId: string; + organizationId: string; + taskSlug: string; + baseTimeMs: number; +}; + +function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { + const { + traceId, + rootSpanId, + runId, + environmentId, + projectId, + organizationId, + taskSlug, + baseTimeMs, + } = params; + + const events: CreateEventInput[] = []; + const runTags = ["user:seed_user_42", "chat:seed_session"]; + + // Timing cursor — each span advances this + let cursor = baseTimeMs; + function next(durationMs: number) { + const start = cursor; + cursor += durationMs + 50; // 50ms gap between spans + return { startMs: start, durationMs }; + } + + function makeEvent(opts: { + message: string; + spanId: string; + parentId: string | undefined; + startMs: number; + durationMs: number; + properties: Record; + style?: Record; + attemptNumber?: number; + }): CreateEventInput { + const startNs = BigInt(opts.startMs) * BigInt(1_000_000); + const durationNs = opts.durationMs * 1_000_000; + return { + traceId, + spanId: opts.spanId, + parentId: opts.parentId, + message: opts.message, + kind: "INTERNAL" as any, + status: "OK" as any, + level: "TRACE" as any, + startTime: startNs, + duration: durationNs, + isError: false, + isPartial: false, + isCancelled: false, + isDebug: false, + runId, + environmentId, + projectId, + organizationId, + taskSlug, + properties: opts.properties, + metadata: undefined, + style: opts.style as any, + events: undefined, + runTags, + attemptNumber: opts.attemptNumber, + }; + } + + // --- Shared prompt content --- + const userMessage = "What is the current Federal Reserve interest rate?"; + const systemPrompt = "You are a helpful financial assistant with access to web search tools."; + const assistantResponse = + "The current Federal Reserve interest rate target range is 4.25% to 4.50%. This was set by the FOMC at their most recent meeting."; + const toolCallResult = JSON.stringify({ + status: 200, + contentType: "text/html", + body: "...Federal Reserve maintains the target range for the federal funds rate at 4-1/4 to 4-1/2 percent...", + truncated: true, + }); + const promptMessages = JSON.stringify([ + { role: "system", content: systemPrompt }, + { role: "user", content: userMessage }, + ]); + const toolDefs = JSON.stringify([ + JSON.stringify({ + type: "function", + name: "webSearch", + description: "Search the web for information", + inputSchema: { + type: "object", + properties: { + query: { type: "string" }, + num: { type: "number" }, + }, + required: ["query"], + }, + }), + ]); + const toolCallsJson = JSON.stringify([ + { + id: "call_seed_001", + type: "function", + function: { + name: "webSearch", + arguments: '{"query":"federal reserve interest rate 2024","num":5}', + }, + }, + ]); + + // --- Span IDs --- + const attemptId = generateSpanId(); + const runFnId = generateSpanId(); + + // streamText sub-tree IDs + const streamWrapId = generateSpanId(); + const stream1Id = generateSpanId(); + const toolCall1Id = generateSpanId(); + const stream2Id = generateSpanId(); + + // generateText sub-tree IDs (Anthropic with cache) + const genTextWrapId = generateSpanId(); + const genTextDoId = generateSpanId(); + const toolCall2Id = generateSpanId(); + + // generateObject sub-tree IDs (gateway → xAI) + const genObjWrapId = generateSpanId(); + const genObjDoId = generateSpanId(); + + // generateObject sub-tree IDs (Google Gemini) + const genObjGeminiWrapId = generateSpanId(); + const genObjGeminiDoId = generateSpanId(); + + // ===================================================================== + // Structural spans: root → attempt → run() + // ===================================================================== + const rootStart = baseTimeMs; + const totalDuration = 120_000; // 2 minutes to cover all ~18 scenarios + + events.push( + makeEvent({ + message: taskSlug, + spanId: rootSpanId, + parentId: undefined, + startMs: rootStart, + durationMs: totalDuration, + properties: {}, + }) + ); + + events.push( + makeEvent({ + message: "Attempt 1", + spanId: attemptId, + parentId: rootSpanId, + startMs: rootStart + 30, + durationMs: totalDuration - 60, + properties: { "$entity.type": "attempt" }, + style: { icon: "attempt", variant: "cold" }, + attemptNumber: 1, + }) + ); + + events.push( + makeEvent({ + message: "run()", + spanId: runFnId, + parentId: attemptId, + startMs: rootStart + 60, + durationMs: totalDuration - 120, + properties: {}, + style: { icon: "task-fn-run" }, + attemptNumber: 1, + }) + ); + + // ===================================================================== + // 1) ai.streamText — OpenAI gpt-4o-mini with tool use (2 LLM calls) + // ===================================================================== + cursor = rootStart + 100; + const stWrap = next(9_500); + + events.push( + makeEvent({ + message: "ai.streamText", + spanId: streamWrapId, + parentId: runFnId, + ...stWrap, + properties: { + "ai.operationId": "ai.streamText", + "ai.model.id": "gpt-4o-mini", + "ai.model.provider": "openai.responses", + "ai.response.finishReason": "stop", + "ai.response.text": assistantResponse, + "ai.usage.inputTokens": 807, + "ai.usage.outputTokens": 242, + "ai.usage.totalTokens": 1049, + "ai.telemetry.metadata.userId": "seed_user_42", + "ai.telemetry.functionId": "ai-chat", + "operation.name": "ai.streamText", + }, + }) + ); + + cursor = stWrap.startMs + 50; + const st1 = next(2_500); + events.push( + makeEvent({ + message: "ai.streamText.doStream", + spanId: stream1Id, + parentId: streamWrapId, + ...st1, + properties: { + "gen_ai.system": "openai.responses", + "gen_ai.request.model": "gpt-4o-mini", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.usage.input_tokens": 284, + "gen_ai.usage.output_tokens": 55, + "ai.model.id": "gpt-4o-mini", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.streamText.doStream", + "ai.prompt.messages": promptMessages, + "ai.prompt.tools": toolDefs, + "ai.prompt.toolChoice": '{"type":"auto"}', + "ai.settings.maxRetries": 2, + "ai.response.finishReason": "tool-calls", + "ai.response.toolCalls": toolCallsJson, + "ai.response.text": "", + "ai.response.id": "resp_seed_001", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.msToFirstChunk": 891.37, + "ai.response.msToFinish": 2321.12, + "ai.response.timestamp": new Date(st1.startMs + st1.durationMs).toISOString(), + "ai.usage.inputTokens": 284, + "ai.usage.outputTokens": 55, + "ai.usage.totalTokens": 339, + "ai.telemetry.metadata.userId": "seed_user_42", + "ai.telemetry.functionId": "ai-chat", + "operation.name": "ai.streamText.doStream", + }, + }) + ); + + const tc1 = next(3_000); + events.push( + makeEvent({ + message: "ai.toolCall", + spanId: toolCall1Id, + parentId: streamWrapId, + ...tc1, + properties: { + "ai.operationId": "ai.toolCall", + "ai.toolCall.name": "webSearch", + "ai.toolCall.id": "call_seed_001", + "ai.toolCall.args": '{"query":"federal reserve interest rate 2024","num":5}', + "ai.toolCall.result": toolCallResult, + "operation.name": "ai.toolCall", + }, + }) + ); + + const st2 = next(3_500); + events.push( + makeEvent({ + message: "ai.streamText.doStream", + spanId: stream2Id, + parentId: streamWrapId, + ...st2, + properties: { + "gen_ai.system": "openai.responses", + "gen_ai.request.model": "gpt-4o-mini", + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.usage.input_tokens": 523, + "gen_ai.usage.output_tokens": 187, + "ai.model.id": "gpt-4o-mini", + "ai.model.provider": "openai.responses", + "ai.operationId": "ai.streamText.doStream", + "ai.prompt.messages": promptMessages, + "ai.settings.maxRetries": 2, + "ai.response.finishReason": "stop", + "ai.response.text": assistantResponse, + "ai.response.reasoning": + "Let me analyze the Federal Reserve data to provide the current rate.", + "ai.response.id": "resp_seed_002", + "ai.response.model": "gpt-4o-mini-2024-07-18", + "ai.response.msToFirstChunk": 672.45, + "ai.response.msToFinish": 3412.89, + "ai.response.timestamp": new Date(st2.startMs + st2.durationMs).toISOString(), + "ai.usage.inputTokens": 523, + "ai.usage.outputTokens": 187, + "ai.usage.totalTokens": 710, + "ai.usage.reasoningTokens": 42, + "ai.telemetry.metadata.userId": "seed_user_42", + "ai.telemetry.functionId": "ai-chat", + "operation.name": "ai.streamText.doStream", + }, + }) + ); + + // ===================================================================== + // 2) ai.generateText — Anthropic claude-haiku-4-5 with tool call + cache + // ===================================================================== + const gtWrap = next(4_200); + + events.push( + makeEvent({ + message: "ai.generateText", + spanId: genTextWrapId, + parentId: runFnId, + ...gtWrap, + properties: { + "ai.operationId": "ai.generateText", + "ai.model.id": "claude-haiku-4-5", + "ai.model.provider": "anthropic.messages", + "ai.response.finishReason": "stop", + "ai.response.text": "Based on the search results, the current rate is 4.25%-4.50%.", + "ai.usage.promptTokens": 9951, + "ai.usage.completionTokens": 803, + "ai.telemetry.metadata.agentName": "research-agent", + "ai.telemetry.functionId": "ai-chat", + "operation.name": "ai.generateText", + }, + }) + ); + + cursor = gtWrap.startMs + 50; + const gtDo = next(3_200); + events.push( + makeEvent({ + message: "ai.generateText.doGenerate", + spanId: genTextDoId, + parentId: genTextWrapId, + ...gtDo, + properties: { + "gen_ai.system": "anthropic.messages", + "gen_ai.request.model": "claude-haiku-4-5", + "gen_ai.response.model": "claude-haiku-4-5-20251001", + "gen_ai.usage.input_tokens": 9951, + "gen_ai.usage.output_tokens": 803, + "gen_ai.usage.cache_read_input_tokens": 8200, + "gen_ai.usage.cache_creation_input_tokens": 1751, + "ai.model.id": "claude-haiku-4-5", + "ai.model.provider": "anthropic.messages", + "ai.operationId": "ai.generateText.doGenerate", + "ai.prompt.messages": promptMessages, + "ai.prompt.toolChoice": '{"type":"auto"}', + "ai.settings.maxRetries": 2, + "ai.response.finishReason": "tool-calls", + "ai.response.id": "msg_seed_003", + "ai.response.model": "claude-haiku-4-5-20251001", + "ai.response.text": + "I'll search for the latest Federal Reserve interest rate information.", + "ai.response.toolCalls": JSON.stringify([ + { + toolCallId: "toolu_seed_001", + toolName: "webSearch", + input: '{"query":"federal reserve interest rate current"}', + }, + ]), + "ai.response.providerMetadata": JSON.stringify({ + anthropic: { + usage: { + input_tokens: 9951, + output_tokens: 803, + cache_creation_input_tokens: 1751, + cache_read_input_tokens: 8200, + service_tier: "standard", + }, + }, + }), + "ai.response.timestamp": new Date(gtDo.startMs + gtDo.durationMs).toISOString(), + "ai.usage.promptTokens": 9951, + "ai.usage.completionTokens": 803, + "ai.telemetry.metadata.agentName": "research-agent", + "ai.telemetry.functionId": "ai-chat", + "operation.name": "ai.generateText.doGenerate", + }, + }) + ); + + const tc2 = next(500); + events.push( + makeEvent({ + message: "ai.toolCall", + spanId: toolCall2Id, + parentId: genTextWrapId, + ...tc2, + properties: { + "ai.operationId": "ai.toolCall", + "ai.toolCall.name": "webSearch", + "ai.toolCall.id": "toolu_seed_001", + "ai.toolCall.args": '{"query":"federal reserve interest rate current"}', + "ai.toolCall.result": + '[{"title":"Federal Reserve Board - Policy Rate","link":"https://federalreserve.gov/rates","snippet":"The target range is 4.25% to 4.50%"}]', + "operation.name": "ai.toolCall", + "resource.name": "ai-chat", + }, + }) + ); + + // ===================================================================== + // 3) ai.generateObject — Gateway → xAI/grok with structured output + // ===================================================================== + const goWrap = next(1_800); + + events.push( + makeEvent({ + message: "ai.generateObject", + spanId: genObjWrapId, + parentId: runFnId, + ...goWrap, + properties: { + "ai.operationId": "ai.generateObject", + "ai.model.id": "xai/grok-4.1-fast-non-reasoning", + "ai.model.provider": "gateway", + "ai.response.finishReason": "stop", + "ai.response.object": JSON.stringify({ + summary: "Fed rate at 4.25%-4.50%", + confidence: 0.95, + sources: ["federalreserve.gov"], + }), + "ai.telemetry.metadata.model": "xai/grok-4.1-fast-non-reasoning", + "ai.telemetry.metadata.schemaType": "schema", + "ai.telemetry.functionId": "generateObject", + "operation.name": "ai.generateObject", + }, + }) + ); + + cursor = goWrap.startMs + 50; + const goDo = next(1_600); + events.push( + makeEvent({ + message: "ai.generateObject.doGenerate", + spanId: genObjDoId, + parentId: genObjWrapId, + ...goDo, + properties: { + "gen_ai.system": "gateway", + "gen_ai.request.model": "xai/grok-4.1-fast-non-reasoning", + "gen_ai.response.model": "xai/grok-4.1-fast-non-reasoning", + "gen_ai.usage.input_tokens": 1629, + "gen_ai.usage.output_tokens": 158, + "ai.model.id": "xai/grok-4.1-fast-non-reasoning", + "ai.model.provider": "gateway", + "ai.operationId": "ai.generateObject.doGenerate", + "ai.prompt.messages": promptMessages, + "ai.settings.maxRetries": 3, + "ai.response.finishReason": "stop", + "ai.response.id": "aiobj_seed_001", + "ai.response.model": "xai/grok-4.1-fast-non-reasoning", + "ai.response.object": JSON.stringify({ + summary: "Fed rate at 4.25%-4.50%", + confidence: 0.95, + sources: ["federalreserve.gov"], + }), + "ai.response.providerMetadata": JSON.stringify({ + gateway: { + routing: { + originalModelId: "xai/grok-4.1-fast-non-reasoning", + resolvedProvider: "xai", + canonicalSlug: "xai/grok-4.1-fast-non-reasoning", + finalProvider: "xai", + modelAttemptCount: 1, + }, + cost: "0.0002905", + generationId: "gen_seed_001", + }, + }), + "ai.response.timestamp": new Date(goDo.startMs + goDo.durationMs).toISOString(), + "ai.usage.completionTokens": 158, + "ai.usage.promptTokens": 1629, + "ai.request.headers.user-agent": "ai/5.0.60", + "operation.name": "ai.generateObject.doGenerate", + }, + }) + ); + + // ===================================================================== + // 4) ai.generateObject — Google Gemini (generative-ai) with thinking tokens + // ===================================================================== + const goGemWrap = next(2_200); + + events.push( + makeEvent({ + message: "ai.generateObject", + spanId: genObjGeminiWrapId, + parentId: runFnId, + ...goGemWrap, + properties: { + "ai.operationId": "ai.generateObject", + "ai.model.id": "gemini-2.5-flash", + "ai.model.provider": "google.generative-ai", + "ai.response.finishReason": "stop", + "ai.response.object": JSON.stringify({ + category: "financial_data", + label: "interest_rate", + }), + "ai.telemetry.functionId": "classify-content", + "operation.name": "ai.generateObject", + }, + }) + ); + + cursor = goGemWrap.startMs + 50; + const goGemDo = next(2_000); + events.push( + makeEvent({ + message: "ai.generateObject.doGenerate", + spanId: genObjGeminiDoId, + parentId: genObjGeminiWrapId, + ...goGemDo, + properties: { + "gen_ai.system": "google.generative-ai", + "gen_ai.request.model": "gemini-2.5-flash", + "gen_ai.response.model": "gemini-2.5-flash", + "gen_ai.usage.input_tokens": 898, + "gen_ai.usage.output_tokens": 521, + "ai.model.id": "gemini-2.5-flash", + "ai.model.provider": "google.generative-ai", + "ai.operationId": "ai.generateObject.doGenerate", + "ai.prompt.messages": JSON.stringify([ + { + role: "user", + content: [ + { + type: "text", + text: "Classify this content: Federal Reserve interest rate analysis", + }, + ], + }, + ]), + "ai.settings.maxRetries": 3, + "ai.response.finishReason": "stop", + "ai.response.id": "aiobj_seed_gemini", + "ai.response.model": "gemini-2.5-flash", + "ai.response.object": JSON.stringify({ + category: "financial_data", + label: "interest_rate", + }), + "ai.response.providerMetadata": JSON.stringify({ + google: { + usageMetadata: { + thoughtsTokenCount: 510, + promptTokenCount: 898, + candidatesTokenCount: 11, + totalTokenCount: 1419, + }, + }, + }), + "ai.response.timestamp": new Date(goGemDo.startMs + goGemDo.durationMs).toISOString(), + "ai.usage.completionTokens": 521, + "ai.usage.promptTokens": 898, + "operation.name": "ai.generateObject.doGenerate", + }, + }) + ); + + // ===================================================================== + // Helper: add a wrapper + doGenerate/doStream pair + // ===================================================================== + function addLlmPair(opts: { + wrapperMsg: string; // e.g. "ai.generateText" + doMsg: string; // e.g. "ai.generateText.doGenerate" + system: string; + reqModel: string; + respModel: string; + inputTokens: number; + outputTokens: number; + finishReason: string; + wrapperDurationMs: number; + doDurationMs: number; + responseText?: string; + responseObject?: string; + responseReasoning?: string; + toolCallsJson?: string; + providerMetadata?: Record; + telemetryMetadata?: Record; + settings?: Record; + /** Use completionTokens/promptTokens instead of inputTokens/outputTokens */ + useCompletionStyle?: boolean; + cacheReadTokens?: number; + cacheCreationTokens?: number; + reasoningTokens?: number; + extraDoProps?: Record; + }) { + const wId = generateSpanId(); + const dId = generateSpanId(); + + const wrap = next(opts.wrapperDurationMs); + const wrapperProps: Record = { + "ai.operationId": opts.wrapperMsg, + "ai.model.id": opts.reqModel, + "ai.model.provider": opts.system, + "ai.response.finishReason": opts.finishReason, + "operation.name": opts.wrapperMsg, + }; + if (opts.responseText) wrapperProps["ai.response.text"] = opts.responseText; + if (opts.responseObject) wrapperProps["ai.response.object"] = opts.responseObject; + if (opts.telemetryMetadata) { + for (const [k, v] of Object.entries(opts.telemetryMetadata)) { + wrapperProps[`ai.telemetry.metadata.${k}`] = v; + } + } + + events.push(makeEvent({ message: opts.wrapperMsg, spanId: wId, parentId: runFnId, ...wrap, properties: wrapperProps })); + + cursor = wrap.startMs + 50; + const doTiming = next(opts.doDurationMs); + + const doProps: Record = { + "gen_ai.system": opts.system, + "gen_ai.request.model": opts.reqModel, + "gen_ai.response.model": opts.respModel, + "gen_ai.usage.input_tokens": opts.inputTokens, + "gen_ai.usage.output_tokens": opts.outputTokens, + "ai.model.id": opts.reqModel, + "ai.model.provider": opts.system, + "ai.operationId": opts.doMsg, + "ai.prompt.messages": promptMessages, + "ai.response.finishReason": opts.finishReason, + "ai.response.id": `resp_seed_${generateSpanId().slice(0, 8)}`, + "ai.response.model": opts.respModel, + "ai.response.timestamp": new Date(doTiming.startMs + doTiming.durationMs).toISOString(), + "operation.name": opts.doMsg, + }; + + // Token style + if (opts.useCompletionStyle) { + doProps["ai.usage.completionTokens"] = opts.outputTokens; + doProps["ai.usage.promptTokens"] = opts.inputTokens; + } else { + doProps["ai.usage.inputTokens"] = opts.inputTokens; + doProps["ai.usage.outputTokens"] = opts.outputTokens; + doProps["ai.usage.totalTokens"] = opts.inputTokens + opts.outputTokens; + } + + if (opts.responseText) doProps["ai.response.text"] = opts.responseText; + if (opts.responseObject) doProps["ai.response.object"] = opts.responseObject; + if (opts.responseReasoning) doProps["ai.response.reasoning"] = opts.responseReasoning; + if (opts.toolCallsJson) doProps["ai.response.toolCalls"] = opts.toolCallsJson; + if (opts.cacheReadTokens) { + doProps["gen_ai.usage.cache_read_input_tokens"] = opts.cacheReadTokens; + } + if (opts.cacheCreationTokens) { + doProps["gen_ai.usage.cache_creation_input_tokens"] = opts.cacheCreationTokens; + } + if (opts.reasoningTokens) { + doProps["ai.usage.reasoningTokens"] = opts.reasoningTokens; + } + if (opts.providerMetadata) { + doProps["ai.response.providerMetadata"] = JSON.stringify(opts.providerMetadata); + } + if (opts.settings) { + for (const [k, v] of Object.entries(opts.settings)) { + doProps[`ai.settings.${k}`] = v; + } + } + if (opts.telemetryMetadata) { + for (const [k, v] of Object.entries(opts.telemetryMetadata)) { + doProps[`ai.telemetry.metadata.${k}`] = v; + } + } + if (opts.extraDoProps) Object.assign(doProps, opts.extraDoProps); + + events.push(makeEvent({ message: opts.doMsg, spanId: dId, parentId: wId, ...doTiming, properties: doProps })); + + return { wrapperId: wId, doId: dId }; + } + + // Helper: add a tool call span + function addToolCall(parentId: string, name: string, args: string, result: string, durationMs = 500) { + const id = generateSpanId(); + const timing = next(durationMs); + events.push(makeEvent({ + message: "ai.toolCall", + spanId: id, + parentId, + ...timing, + properties: { + "ai.operationId": "ai.toolCall", + "ai.toolCall.name": name, + "ai.toolCall.id": `call_${generateSpanId().slice(0, 8)}`, + "ai.toolCall.args": args, + "ai.toolCall.result": result, + "operation.name": "ai.toolCall", + }, + })); + return id; + } + + // ===================================================================== + // 5) Gateway → Mistral mistral-large-3 + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "gateway", + reqModel: "mistral/mistral-large-3", + respModel: "mistral/mistral-large-3", + inputTokens: 1179, + outputTokens: 48, + finishReason: "stop", + wrapperDurationMs: 1_400, + doDurationMs: 1_200, + responseText: "The document discusses quarterly earnings guidance for tech sector.", + useCompletionStyle: true, + providerMetadata: { + gateway: { + routing: { + originalModelId: "mistral/mistral-large-3", + resolvedProvider: "mistral", + resolvedProviderApiModelId: "mistral-large-latest", + canonicalSlug: "mistral/mistral-large-3", + finalProvider: "mistral", + modelAttemptCount: 1, + }, + cost: "0.0006615", + marketCost: "0.0006615", + generationId: "gen_seed_mistral_001", + }, + }, + extraDoProps: { "ai.request.headers.user-agent": "ai/5.0.60" }, + }); + + // ===================================================================== + // 6) Gateway → OpenAI gpt-5-mini (with fallback metadata) + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "gateway", + reqModel: "openai/gpt-5-mini", + respModel: "openai/gpt-5-mini", + inputTokens: 2450, + outputTokens: 312, + finishReason: "stop", + wrapperDurationMs: 5_000, + doDurationMs: 4_800, + responseText: "NO", + useCompletionStyle: true, + providerMetadata: { + openai: { responseId: "resp_seed_gw_openai", serviceTier: "default" }, + gateway: { + routing: { + originalModelId: "openai/gpt-5-mini", + resolvedProvider: "openai", + resolvedProviderApiModelId: "gpt-5-mini-2025-08-07", + canonicalSlug: "openai/gpt-5-mini", + finalProvider: "openai", + fallbacksAvailable: ["azure"], + planningReasoning: "System credentials planned for: openai, azure. Total execution order: openai(system) → azure(system)", + modelAttemptCount: 1, + }, + cost: "0.000482", + generationId: "gen_seed_gpt5mini_001", + }, + }, + extraDoProps: { "ai.request.headers.user-agent": "ai/6.0.49" }, + }); + + // ===================================================================== + // 7) Gateway → DeepSeek deepseek-v3.2 (tool-calls) + // ===================================================================== + const ds = addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "gateway", + reqModel: "deepseek/deepseek-v3.2", + respModel: "deepseek/deepseek-v3.2", + inputTokens: 3200, + outputTokens: 420, + finishReason: "tool-calls", + wrapperDurationMs: 2_800, + doDurationMs: 2_500, + responseObject: JSON.stringify({ action: "search", query: "fed rate history" }), + useCompletionStyle: true, + providerMetadata: { + gateway: { + routing: { + originalModelId: "deepseek/deepseek-v3.2", + resolvedProvider: "deepseek", + canonicalSlug: "deepseek/deepseek-v3.2", + finalProvider: "deepseek", + modelAttemptCount: 1, + }, + cost: "0.000156", + generationId: "gen_seed_deepseek_001", + }, + }, + }); + addToolCall(ds.wrapperId, "classifyContent", '{"text":"Federal Reserve rate analysis"}', '{"category":"finance","confidence":0.98}'); + + // ===================================================================== + // 8) Gateway → Anthropic claude-haiku via gateway prefix + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "gateway", + reqModel: "anthropic/claude-haiku-4-5-20251001", + respModel: "anthropic/claude-haiku-4-5-20251001", + inputTokens: 5400, + outputTokens: 220, + finishReason: "stop", + wrapperDurationMs: 1_800, + doDurationMs: 1_500, + responseText: "The content appears to be a standard financial news article. Classification: SAFE.", + useCompletionStyle: true, + providerMetadata: { + gateway: { + routing: { + originalModelId: "anthropic/claude-haiku-4-5-20251001", + resolvedProvider: "anthropic", + canonicalSlug: "anthropic/claude-haiku-4-5-20251001", + finalProvider: "anthropic", + modelAttemptCount: 1, + }, + cost: "0.00312", + generationId: "gen_seed_gw_anthropic_001", + }, + }, + }); + + // ===================================================================== + // 9) Gateway → Google gemini-3-flash-preview (structured output) + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "gateway", + reqModel: "google/gemini-3-flash-preview", + respModel: "google/gemini-3-flash-preview", + inputTokens: 720, + outputTokens: 85, + finishReason: "stop", + wrapperDurationMs: 1_200, + doDurationMs: 1_000, + responseObject: JSON.stringify({ sentiment: "neutral", topics: ["monetary_policy", "interest_rates"] }), + useCompletionStyle: true, + providerMetadata: { + gateway: { + routing: { + originalModelId: "google/gemini-3-flash-preview", + resolvedProvider: "google", + canonicalSlug: "google/gemini-3-flash-preview", + finalProvider: "google", + modelAttemptCount: 1, + }, + cost: "0.0000803", + generationId: "gen_seed_gw_gemini_001", + }, + }, + }); + + // ===================================================================== + // 10) OpenRouter → x-ai/grok-4-fast (with reasoning_details) + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "openrouter", + reqModel: "x-ai/grok-4-fast", + respModel: "x-ai/grok-4-fast", + inputTokens: 375, + outputTokens: 226, + finishReason: "stop", + wrapperDurationMs: 1_600, + doDurationMs: 1_400, + responseObject: JSON.stringify({ hook: "Breaking: Fed holds rates steady", isValidHook: true }), + useCompletionStyle: true, + telemetryMetadata: { model: "x-ai/grok-4-fast", schemaType: "schema", temperature: "1" }, + settings: { maxRetries: 2, temperature: 1 }, + providerMetadata: { + openrouter: { + provider: "xAI", + reasoning_details: [{ type: "reasoning.encrypted", data: "encrypted_seed_data..." }], + usage: { + promptTokens: 375, + promptTokensDetails: { cachedTokens: 343 }, + completionTokens: 226, + completionTokensDetails: { reasoningTokens: 210 }, + totalTokens: 601, + cost: 0.0001351845, + costDetails: { upstreamInferenceCost: 0.00013655 }, + }, + }, + }, + }); + + // ===================================================================== + // 11) OpenRouter → google/gemini-2.5-flash + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "openrouter", + reqModel: "google/gemini-2.5-flash", + respModel: "google/gemini-2.5-flash", + inputTokens: 1840, + outputTokens: 320, + finishReason: "stop", + wrapperDurationMs: 2_000, + doDurationMs: 1_800, + responseText: "Based on the latest FOMC minutes, the committee voted unanimously to maintain rates.", + useCompletionStyle: true, + providerMetadata: { + openrouter: { + provider: "Google AI Studio", + usage: { + promptTokens: 1840, + completionTokens: 320, + totalTokens: 2160, + cost: 0.000264, + costDetails: { upstreamInferenceCost: 0.000232 }, + }, + }, + }, + }); + + // ===================================================================== + // 12) OpenRouter → openai/gpt-4.1-mini (req ≠ resp model name) + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "openrouter", + reqModel: "openai/gpt-4.1-mini", + respModel: "openai/gpt-4.1-mini-2025-04-14", + inputTokens: 890, + outputTokens: 145, + finishReason: "stop", + wrapperDurationMs: 1_400, + doDurationMs: 1_200, + responseObject: JSON.stringify({ summary: "Rate unchanged at 4.25-4.50%", date: "2024-12-18" }), + useCompletionStyle: true, + providerMetadata: { + openrouter: { + provider: "OpenAI", + usage: { + promptTokens: 890, + completionTokens: 145, + totalTokens: 1035, + cost: 0.0000518, + }, + }, + }, + }); + + // ===================================================================== + // 13) Azure → gpt-5 with tool-calls + // ===================================================================== + const az = addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "azure.responses", + reqModel: "gpt-5-2025-08-07", + respModel: "gpt-5-2025-08-07", + inputTokens: 2038, + outputTokens: 239, + finishReason: "tool-calls", + wrapperDurationMs: 3_500, + doDurationMs: 3_000, + responseText: "Let me look up the latest rate decision.", + toolCallsJson: JSON.stringify([{ + toolCallId: "call_azure_001", + toolName: "lookupRate", + input: '{"source":"federal_reserve","metric":"funds_rate"}', + }]), + providerMetadata: { + azure: { responseId: "resp_seed_azure_001", serviceTier: "default" }, + }, + }); + addToolCall(az.wrapperId, "lookupRate", '{"source":"federal_reserve","metric":"funds_rate"}', '{"rate":"4.25-4.50%","effectiveDate":"2024-12-18"}'); + + // ===================================================================== + // 14) Perplexity → sonar-pro + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "perplexity", + reqModel: "sonar-pro", + respModel: "sonar-pro", + inputTokens: 151, + outputTokens: 428, + finishReason: "stop", + wrapperDurationMs: 4_500, + doDurationMs: 4_200, + responseText: "According to the Federal Reserve's most recent announcement on December 18, 2024, the federal funds rate target range was maintained at 4.25% to 4.50%. This decision was made during the December FOMC meeting.", + }); + + // ===================================================================== + // 15) openai.chat → gpt-4o-mini (legacy chat completions, mode: "tool") + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "openai.chat", + reqModel: "gpt-4o-mini", + respModel: "gpt-4o-mini-2024-07-18", + inputTokens: 573, + outputTokens: 11, + finishReason: "stop", + wrapperDurationMs: 800, + doDurationMs: 600, + responseObject: JSON.stringify({ title: "Fed Rate Hold", emoji: "🏦" }), + settings: { maxRetries: 2, mode: "tool", temperature: 0.3 }, + providerMetadata: { + openai: { reasoningTokens: 0, cachedPromptTokens: 0 }, + }, + }); + + // ===================================================================== + // 16) Anthropic claude-sonnet-4-5 → streamText with reasoning + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.streamText", + doMsg: "ai.streamText.doStream", + system: "anthropic.messages", + reqModel: "claude-sonnet-4-5-20250929", + respModel: "claude-sonnet-4-5-20250929", + inputTokens: 15200, + outputTokens: 2840, + finishReason: "stop", + wrapperDurationMs: 12_000, + doDurationMs: 11_500, + responseText: "The Federal Reserve has maintained its target range for the federal funds rate at 4.25% to 4.50% since December 2024. This represents a pause in the rate-cutting cycle that began in September 2024. The FOMC has indicated it will continue to assess incoming data, the evolving outlook, and the balance of risks when considering further adjustments.", + responseReasoning: "The user is asking about the current Federal Reserve interest rate. Let me provide a comprehensive answer based on the most recent FOMC decision. I should include context about the rate trajectory and forward guidance.", + cacheReadTokens: 12400, + cacheCreationTokens: 2800, + providerMetadata: { + anthropic: { + usage: { + input_tokens: 15200, + output_tokens: 2840, + cache_creation_input_tokens: 2800, + cache_read_input_tokens: 12400, + service_tier: "standard", + inference_geo: "us-east-1", + }, + }, + }, + }); + + // ===================================================================== + // 17) google.vertex.chat → gemini-3.1-pro-preview with tool-calls + // ===================================================================== + const vt = addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "google.vertex.chat", + reqModel: "gemini-3.1-pro-preview", + respModel: "gemini-3.1-pro-preview", + inputTokens: 4200, + outputTokens: 680, + finishReason: "tool-calls", + wrapperDurationMs: 6_000, + doDurationMs: 5_500, + responseText: "I'll search for the latest FOMC decision and rate information.", + toolCallsJson: JSON.stringify([{ + toolCallId: "call_vertex_001", + toolName: "searchFOMC", + input: '{"query":"latest FOMC decision december 2024"}', + }]), + providerMetadata: { + google: { + usageMetadata: { + thoughtsTokenCount: 320, + promptTokenCount: 4200, + candidatesTokenCount: 680, + totalTokenCount: 5200, + }, + }, + }, + }); + addToolCall(vt.wrapperId, "searchFOMC", '{"query":"latest FOMC decision december 2024"}', '{"decision":"hold","rate":"4.25-4.50%","date":"2024-12-18","vote":"unanimous"}', 800); + + // ===================================================================== + // 18) openai.responses → gpt-5.4 with reasoning tokens + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.streamText", + doMsg: "ai.streamText.doStream", + system: "openai.responses", + reqModel: "gpt-5.4", + respModel: "gpt-5.4-2026-03-05", + inputTokens: 8900, + outputTokens: 1250, + finishReason: "stop", + wrapperDurationMs: 8_000, + doDurationMs: 7_500, + responseText: "The Federal Reserve's current target range for the federal funds rate is 4.25% to 4.50%, established at the December 2024 FOMC meeting. The committee has signaled a cautious approach to further rate cuts, citing persistent inflation concerns.", + responseReasoning: "I need to provide accurate, up-to-date information about the Federal Reserve interest rate. The last FOMC meeting was in December 2024 where they held rates steady after three consecutive cuts.", + reasoningTokens: 516, + providerMetadata: { + openai: { + responseId: "resp_seed_gpt54_001", + serviceTier: "default", + }, + }, + extraDoProps: { + "ai.response.msToFirstChunk": 1842.5, + "ai.response.msToFinish": 7234.8, + "ai.response.avgOutputTokensPerSecond": 172.8, + }, + }); + + // ===================================================================== + // 19) Cerebras cerebras-gpt-13b — no pricing, no provider cost + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "cerebras.chat", + reqModel: "cerebras-gpt-13b", + respModel: "cerebras-gpt-13b", + inputTokens: 450, + outputTokens: 120, + finishReason: "stop", + wrapperDurationMs: 600, + doDurationMs: 400, + responseText: "The Federal Reserve rate is currently at 4.25-4.50%.", + }); + + // ===================================================================== + // 20) Amazon Bedrock — no pricing in registry + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateText", + doMsg: "ai.generateText.doGenerate", + system: "amazon-bedrock", + reqModel: "us.anthropic.claude-sonnet-4-20250514-v1:0", + respModel: "us.anthropic.claude-sonnet-4-20250514-v1:0", + inputTokens: 3200, + outputTokens: 890, + finishReason: "stop", + wrapperDurationMs: 4_000, + doDurationMs: 3_500, + responseText: "Based on the latest FOMC statement, the target rate range remains at 4.25% to 4.50%.", + }); + + // ===================================================================== + // 21) Groq — fast inference, no pricing + // ===================================================================== + addLlmPair({ + wrapperMsg: "ai.generateObject", + doMsg: "ai.generateObject.doGenerate", + system: "groq.chat", + reqModel: "llama-4-scout-17b-16e-instruct", + respModel: "llama-4-scout-17b-16e-instruct", + inputTokens: 820, + outputTokens: 95, + finishReason: "stop", + wrapperDurationMs: 300, + doDurationMs: 200, + responseObject: JSON.stringify({ rate: "4.25-4.50%", source: "FOMC", date: "2024-12-18" }), + }); + + return events; +} + +// --------------------------------------------------------------------------- + +seedAiSpans() + .catch((e) => { + console.error("Seed failed:"); + console.error(e); + process.exit(1); + }) + .finally(async () => { + await prisma.$disconnect(); + }); diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts index 5b340eb4034..9ffb14eba49 100644 --- a/internal-packages/llm-pricing/src/registry.ts +++ b/internal-packages/llm-pricing/src/registry.ts @@ -103,6 +103,18 @@ export class ModelPricingRegistry { } } + // Fallback: strip provider prefix (e.g. "mistral/mistral-large-3" → "mistral-large-3") + // Gateway and OpenRouter prepend the provider to the model name. + if (responseModel.includes("/")) { + const stripped = responseModel.split("/").slice(1).join("/"); + for (const { regex, model } of this._patterns) { + if (regex.test(stripped)) { + this._exactMatchCache.set(responseModel, model); + return model; + } + } + } + // Cache miss this._exactMatchCache.set(responseModel, null); return null; From 74fa6cb857727cab79f440594d970fd40847208d Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 13 Mar 2026 11:30:15 +0000 Subject: [PATCH 06/30] Fixed a bunch of coderabbit issues --- .../admin.api.v1.llm-models.$modelId.ts | 89 ++++++++++--------- .../app/routes/admin.api.v1.llm-models.ts | 77 ++++++++-------- .../admin.llm-models.missing.$model.tsx | 6 +- .../services/admin/missingLlmModels.server.ts | 3 +- .../llm-pricing/scripts/sync-model-prices.sh | 2 +- .../llm-pricing/src/defaultPrices.ts | 2 +- internal-packages/llm-pricing/src/seed.ts | 55 ++++++------ 7 files changed, 123 insertions(+), 111 deletions(-) diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts index 8cf132daab9..4e8357c886c 100644 --- a/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts @@ -81,7 +81,13 @@ export async function action({ request, params }: ActionFunctionArgs) { return json({ error: "Method not allowed" }, { status: 405 }); } - const body = await request.json(); + let body: unknown; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + const parsed = UpdateModelSchema.safeParse(body); if (!parsed.success) { @@ -90,59 +96,56 @@ export async function action({ request, params }: ActionFunctionArgs) { const { modelName, matchPattern, startDate, pricingTiers } = parsed.data; - // Validate regex if provided + // Validate regex if provided — strip (?i) POSIX flag since our registry handles it if (matchPattern) { try { - new RegExp(matchPattern); + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); } catch { return json({ error: "Invalid regex in matchPattern" }, { status: 400 }); } } - // Update model fields - const model = await prisma.llmModel.update({ - where: { id: modelId }, - data: { - ...(modelName !== undefined && { modelName }), - ...(matchPattern !== undefined && { matchPattern }), - ...(startDate !== undefined && { startDate: startDate ? new Date(startDate) : null }), - }, - }); - - // If pricing tiers provided, replace them entirely - if (pricingTiers) { - // Delete existing tiers (cascades to prices) - await prisma.llmPricingTier.deleteMany({ where: { modelId } }); - - // Create new tiers - for (const tier of pricingTiers) { - await prisma.llmPricingTier.create({ - data: { - modelId, - name: tier.name, - isDefault: tier.isDefault, - priority: tier.priority, - conditions: tier.conditions, - prices: { - create: Object.entries(tier.prices).map(([usageType, price]) => ({ - modelId, - usageType, - price, - })), + // Update model + tiers atomically + const updated = await prisma.$transaction(async (tx) => { + await tx.llmModel.update({ + where: { id: modelId }, + data: { + ...(modelName !== undefined && { modelName }), + ...(matchPattern !== undefined && { matchPattern }), + ...(startDate !== undefined && { startDate: startDate ? new Date(startDate) : null }), + }, + }); + + if (pricingTiers) { + await tx.llmPricingTier.deleteMany({ where: { modelId } }); + + for (const tier of pricingTiers) { + await tx.llmPricingTier.create({ + data: { + modelId, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId, + usageType, + price, + })), + }, }, - }, - }); + }); + } } - } - const updated = await prisma.llmModel.findUnique({ - where: { id: modelId }, - include: { - pricingTiers: { - include: { prices: true }, - orderBy: { priority: "asc" }, + return tx.llmModel.findUnique({ + where: { id: modelId }, + include: { + pricingTiers: { include: { prices: true }, orderBy: { priority: "asc" } }, }, - }, + }); }); return json({ model: updated }); diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.ts index 706c73549de..6305869c605 100644 --- a/apps/webapp/app/routes/admin.api.v1.llm-models.ts +++ b/apps/webapp/app/routes/admin.api.v1.llm-models.ts @@ -75,7 +75,13 @@ export async function action({ request }: ActionFunctionArgs) { return json({ error: "Method not allowed" }, { status: 405 }); } - const body = await request.json(); + let body: unknown; + try { + body = await request.json(); + } catch { + return json({ error: "Invalid JSON body" }, { status: 400 }); + } + const parsed = CreateModelSchema.safeParse(body); if (!parsed.success) { @@ -84,50 +90,51 @@ export async function action({ request }: ActionFunctionArgs) { const { modelName, matchPattern, startDate, source, pricingTiers } = parsed.data; - // Validate regex pattern + // Validate regex pattern — strip (?i) POSIX flag since our registry handles it try { - new RegExp(matchPattern); + const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern; + new RegExp(testPattern); } catch { return json({ error: "Invalid regex in matchPattern" }, { status: 400 }); } - // Create model first, then tiers with explicit model connection - const model = await prisma.llmModel.create({ - data: { - friendlyId: generateFriendlyId("llm_model"), - modelName, - matchPattern, - startDate: startDate ? new Date(startDate) : null, - source, - }, - }); - - for (const tier of pricingTiers) { - await prisma.llmPricingTier.create({ + // Create model + tiers atomically + const created = await prisma.$transaction(async (tx) => { + const model = await tx.llmModel.create({ data: { - modelId: model.id, - name: tier.name, - isDefault: tier.isDefault, - priority: tier.priority, - conditions: tier.conditions, - prices: { - create: Object.entries(tier.prices).map(([usageType, price]) => ({ - modelId: model.id, - usageType, - price, - })), - }, + friendlyId: generateFriendlyId("llm_model"), + modelName, + matchPattern, + startDate: startDate ? new Date(startDate) : null, + source, }, }); - } - const created = await prisma.llmModel.findUnique({ - where: { id: model.id }, - include: { - pricingTiers: { - include: { prices: true }, + for (const tier of pricingTiers) { + await tx.llmPricingTier.create({ + data: { + modelId: model.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId: model.id, + usageType, + price, + })), + }, + }, + }); + } + + return tx.llmModel.findUnique({ + where: { id: model.id }, + include: { + pricingTiers: { include: { prices: true } }, }, - }, + }); }); return json({ model: created }, { status: 201 }); diff --git a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx index 7b6e83bf939..fafa3c81235 100644 --- a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx +++ b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx @@ -16,7 +16,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const user = await prisma.user.findUnique({ where: { id: userId } }); if (!user?.admin) return redirect("/"); - // Model name is base64url-encoded in the URL param + // Model name is URL-encoded in the URL param const modelName = decodeURIComponent(params.model ?? ""); if (!modelName) throw new Response("Missing model param", { status: 400 }); @@ -142,7 +142,7 @@ export default function AdminMissingModelDetailRoute() {
input: {providerCosts[0].estimatedInputPrice.toExponential(4)} - output: {providerCosts[0].estimatedOutputPrice!.toExponential(4)} + output: {providerCosts[0].estimatedOutputPrice ?? 0.toExponential(4)}
Cross-reference with the provider's pricing page before using these estimates. @@ -457,7 +457,7 @@ The gateway/router is reporting costs for this model. Use these to cross-referen ${providerCosts.map((c) => `- $${c.cost.toFixed(6)} for ${c.inputTokens.toLocaleString()} input + ${c.outputTokens.toLocaleString()} output tokens`).join("\n")}${providerCosts[0].estimatedInputPrice != null ? ` - Estimated per-token rates (rough, assuming ~3x output/input ratio): - input: ${providerCosts[0].estimatedInputPrice.toExponential(4)} (${(providerCosts[0].estimatedInputPrice * 1_000_000).toFixed(4)} $/M) - - output: ${providerCosts[0].estimatedOutputPrice!.toExponential(4)} (${(providerCosts[0].estimatedOutputPrice! * 1_000_000).toFixed(4)} $/M) + - output: ${providerCosts[0].estimatedOutputPrice ?? 0.toExponential(4)} (${(providerCosts[0].estimatedOutputPrice ?? 0 * 1_000_000).toFixed(4)} $/M) - Verify these against the official pricing page before using.` : ""}` : ""}${sampleAttrs ? ` ## Sample span attributes (first span) diff --git a/apps/webapp/app/services/admin/missingLlmModels.server.ts b/apps/webapp/app/services/admin/missingLlmModels.server.ts index 01237ef2b57..0ad06e7e2d9 100644 --- a/apps/webapp/app/services/admin/missingLlmModels.server.ts +++ b/apps/webapp/app/services/admin/missingLlmModels.server.ts @@ -70,7 +70,8 @@ export async function getMissingLlmModels(opts: { // Filter out models that now have pricing in the database (added after spans were inserted). // The registry's match() handles prefix stripping for gateway/openrouter models. - return candidates.filter((c) => !llmPricingRegistry?.match(c.model)); + if (!llmPricingRegistry?.isLoaded) return candidates; + return candidates.filter((c) => !llmPricingRegistry.match(c.model)); } export type MissingModelSample = { diff --git a/internal-packages/llm-pricing/scripts/sync-model-prices.sh b/internal-packages/llm-pricing/scripts/sync-model-prices.sh index 9bb6af11cbb..d72aa6714c6 100755 --- a/internal-packages/llm-pricing/scripts/sync-model-prices.sh +++ b/internal-packages/llm-pricing/scripts/sync-model-prices.sh @@ -50,7 +50,7 @@ echo "Generating defaultPrices.ts..." node -e " const data = JSON.parse(require('fs').readFileSync('$JSON_TARGET', 'utf-8')); const stripped = data.map(e => ({ - modelName: e.modelName, + modelName: e.modelName.trim(), matchPattern: e.matchPattern, startDate: e.createdAt, pricingTiers: e.pricingTiers.map(t => ({ diff --git a/internal-packages/llm-pricing/src/defaultPrices.ts b/internal-packages/llm-pricing/src/defaultPrices.ts index 91b8a6f89f6..689944a6432 100644 --- a/internal-packages/llm-pricing/src/defaultPrices.ts +++ b/internal-packages/llm-pricing/src/defaultPrices.ts @@ -1109,7 +1109,7 @@ export const defaultModelPrices: DefaultModelDefinition[] = [ ] }, { - "modelName": " gpt-4-preview", + "modelName": "gpt-4-preview", "matchPattern": "(?i)^(openai/)?(gpt-4-preview)$", "startDate": "2024-04-23T10:37:17.092Z", "pricingTiers": [ diff --git a/internal-packages/llm-pricing/src/seed.ts b/internal-packages/llm-pricing/src/seed.ts index b4f95373eff..d068c62a66d 100644 --- a/internal-packages/llm-pricing/src/seed.ts +++ b/internal-packages/llm-pricing/src/seed.ts @@ -23,36 +23,37 @@ export async function seedLlmPricing(prisma: PrismaClient): Promise<{ continue; } - // Create model first - const model = await prisma.llmModel.create({ - data: { - friendlyId: generateFriendlyId("llm_model"), - modelName: modelDef.modelName, - matchPattern: modelDef.matchPattern, - startDate: modelDef.startDate ? new Date(modelDef.startDate) : null, - source: "default", - }, - }); - - // Create tiers and prices with explicit model connection - for (const tier of modelDef.pricingTiers) { - await prisma.llmPricingTier.create({ + // Create model + tiers atomically so partial models can't be left behind + await prisma.$transaction(async (tx) => { + const model = await tx.llmModel.create({ data: { - modelId: model.id, - name: tier.name, - isDefault: tier.isDefault, - priority: tier.priority, - conditions: tier.conditions, - prices: { - create: Object.entries(tier.prices).map(([usageType, price]) => ({ - modelId: model.id, - usageType, - price, - })), - }, + friendlyId: generateFriendlyId("llm_model"), + modelName: modelDef.modelName.trim(), + matchPattern: modelDef.matchPattern, + startDate: modelDef.startDate ? new Date(modelDef.startDate) : null, + source: "default", }, }); - } + + for (const tier of modelDef.pricingTiers) { + await tx.llmPricingTier.create({ + data: { + modelId: model.id, + name: tier.name, + isDefault: tier.isDefault, + priority: tier.priority, + conditions: tier.conditions, + prices: { + create: Object.entries(tier.prices).map(([usageType, price]) => ({ + modelId: model.id, + usageType, + price, + })), + }, + }, + }); + } + }); modelsCreated++; } From 8030da7d978908bc8c7a963361f4e01913342723 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 13 Mar 2026 12:27:17 +0000 Subject: [PATCH 07/30] fix some typescript errors --- .../runs/v3/ai/extractAISpanData.ts | 9 +++---- .../app/routes/admin.llm-models.$modelId.tsx | 6 ++--- .../app/routes/admin.llm-models._index.tsx | 4 ++-- .../admin.llm-models.missing.$model.tsx | 24 ++++++++++--------- .../admin.llm-models.missing._index.tsx | 2 +- .../app/routes/admin.llm-models.new.tsx | 6 ++--- .../services/admin/missingLlmModels.server.ts | 5 ++-- .../v3/utils/enrichCreatableEvents.server.ts | 6 ++--- internal-packages/llm-pricing/src/registry.ts | 6 ++--- internal-packages/tsql/src/query/schema.ts | 4 ++++ 10 files changed, 40 insertions(+), 32 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts b/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts index de96c073a28..82c1014d5ab 100644 --- a/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts +++ b/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts @@ -426,7 +426,7 @@ function parseProviderMetadata( ): { serviceTier?: string; resolvedProvider?: string; gatewayCost?: string } | undefined { if (typeof raw !== "string") return undefined; try { - const parsed = JSON.parse(raw); + const parsed = JSON.parse(raw) as Record; if (!parsed || typeof parsed !== "object") return undefined; let serviceTier: string | undefined; @@ -469,12 +469,13 @@ function parseToolChoice(raw: unknown): string | undefined { try { const parsed = JSON.parse(raw); if (typeof parsed === "string") return parsed; - if (parsed && typeof parsed === "object" && typeof parsed.type === "string") { - return parsed.type; + if (parsed && typeof parsed === "object") { + const obj = parsed as Record; + if (typeof obj.type === "string") return obj.type; } return undefined; } catch { - return raw || undefined; + return undefined; } } diff --git a/apps/webapp/app/routes/admin.llm-models.$modelId.tsx b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx index 4e72731cc3e..7f7fadaf436 100644 --- a/apps/webapp/app/routes/admin.llm-models.$modelId.tsx +++ b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx @@ -14,7 +14,7 @@ import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const model = await prisma.llmModel.findUnique({ where: { friendlyId: params.modelId }, @@ -46,7 +46,7 @@ const SaveSchema = z.object({ export async function action({ request, params }: ActionFunctionArgs) { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const friendlyId = params.modelId!; const existing = await prisma.llmModel.findUnique({ where: { friendlyId } }); @@ -89,7 +89,7 @@ export async function action({ request, params }: ActionFunctionArgs) { prices: Record; }>; try { - pricingTiers = JSON.parse(pricingTiersJson); + pricingTiers = JSON.parse(pricingTiersJson) as typeof pricingTiers; } catch { return typedjson({ error: "Invalid pricing tiers JSON" }, { status: 400 }); } diff --git a/apps/webapp/app/routes/admin.llm-models._index.tsx b/apps/webapp/app/routes/admin.llm-models._index.tsx index 2bfa46d0c8a..fb2f6fdc491 100644 --- a/apps/webapp/app/routes/admin.llm-models._index.tsx +++ b/apps/webapp/app/routes/admin.llm-models._index.tsx @@ -33,7 +33,7 @@ const SearchParams = z.object({ export const loader = async ({ request }: LoaderFunctionArgs) => { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const searchParams = createSearchParams(request.url, SearchParams); if (!searchParams.success) throw new Error(searchParams.error); @@ -79,7 +79,7 @@ export const loader = async ({ request }: LoaderFunctionArgs) => { export async function action({ request }: ActionFunctionArgs) { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const formData = await request.formData(); const _action = formData.get("_action"); diff --git a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx index fafa3c81235..78cb1c4fc91 100644 --- a/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx +++ b/apps/webapp/app/routes/admin.llm-models.missing.$model.tsx @@ -14,7 +14,7 @@ import { export const loader = async ({ request, params }: LoaderFunctionArgs) => { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); // Model name is URL-encoded in the URL param const modelName = decodeURIComponent(params.model ?? ""); @@ -142,7 +142,7 @@ export default function AdminMissingModelDetailRoute() {
input: {providerCosts[0].estimatedInputPrice.toExponential(4)} - output: {providerCosts[0].estimatedOutputPrice ?? 0.toExponential(4)} + output: {(providerCosts[0].estimatedOutputPrice ?? 0).toExponential(4)}
Cross-reference with the provider's pricing page before using these estimates. @@ -176,7 +176,7 @@ export default function AdminMissingModelDetailRoute() { const expanded = expandedSpans.has(s.span_id); let parsedAttrs: Record | null = null; try { - parsedAttrs = JSON.parse(s.attributes_text); + parsedAttrs = JSON.parse(s.attributes_text) as Record; } catch { // ignore } @@ -226,7 +226,7 @@ function extractTokenTypes(samples: MissingModelSample[]): TokenTypeSummary[] { for (const s of samples) { let attrs: Record; try { - attrs = JSON.parse(s.attributes_text); + attrs = JSON.parse(s.attributes_text) as Record; } catch { continue; } @@ -300,7 +300,7 @@ function extractProviderCosts(samples: MissingModelSample[]): ProviderCostInfo[] for (const s of samples) { let attrs: Record; try { - attrs = JSON.parse(s.attributes_text); + attrs = JSON.parse(s.attributes_text) as Record; } catch { continue; } @@ -310,7 +310,7 @@ function extractProviderCosts(samples: MissingModelSample[]): ProviderCostInfo[] const aiResponse = getNestedObj(attrs, ["ai", "response"]); const rawMeta = aiResponse?.providerMetadata; if (typeof rawMeta === "string") { - try { providerMeta = JSON.parse(rawMeta); } catch {} + try { providerMeta = JSON.parse(rawMeta) as Record; } catch {} } else if (rawMeta && typeof rawMeta === "object") { providerMeta = rawMeta as Record; } @@ -390,13 +390,15 @@ function buildPrompt(modelName: string, samples: MissingModelSample[], providerC let sampleAttrs = ""; if (samples.length > 0) { try { - const attrs = JSON.parse(samples[0].attributes_text); + const attrs = JSON.parse(samples[0].attributes_text) as Record; + const ai = attrs.ai as Record | undefined; + const aiResponse = (ai?.response ?? {}) as Record; // Extract just the relevant fields const compact: Record = {}; if (attrs.gen_ai) compact.gen_ai = attrs.gen_ai; - if (attrs.ai?.usage) compact["ai.usage"] = attrs.ai.usage; - if (attrs.ai?.response?.providerMetadata) { - compact["ai.response.providerMetadata"] = attrs.ai.response.providerMetadata; + if (ai?.usage) compact["ai.usage"] = ai.usage; + if (aiResponse.providerMetadata) { + compact["ai.response.providerMetadata"] = aiResponse.providerMetadata; } sampleAttrs = JSON.stringify(compact, null, 2); } catch { @@ -457,7 +459,7 @@ The gateway/router is reporting costs for this model. Use these to cross-referen ${providerCosts.map((c) => `- $${c.cost.toFixed(6)} for ${c.inputTokens.toLocaleString()} input + ${c.outputTokens.toLocaleString()} output tokens`).join("\n")}${providerCosts[0].estimatedInputPrice != null ? ` - Estimated per-token rates (rough, assuming ~3x output/input ratio): - input: ${providerCosts[0].estimatedInputPrice.toExponential(4)} (${(providerCosts[0].estimatedInputPrice * 1_000_000).toFixed(4)} $/M) - - output: ${providerCosts[0].estimatedOutputPrice ?? 0.toExponential(4)} (${(providerCosts[0].estimatedOutputPrice ?? 0 * 1_000_000).toFixed(4)} $/M) + - output: ${(providerCosts[0].estimatedOutputPrice ?? 0).toExponential(4)} (${((providerCosts[0].estimatedOutputPrice ?? 0) * 1_000_000).toFixed(4)} $/M) - Verify these against the official pricing page before using.` : ""}` : ""}${sampleAttrs ? ` ## Sample span attributes (first span) diff --git a/apps/webapp/app/routes/admin.llm-models.missing._index.tsx b/apps/webapp/app/routes/admin.llm-models.missing._index.tsx index b778b9a3dc9..fd933cd22e9 100644 --- a/apps/webapp/app/routes/admin.llm-models.missing._index.tsx +++ b/apps/webapp/app/routes/admin.llm-models.missing._index.tsx @@ -33,7 +33,7 @@ const SearchParams = z.object({ export const loader = async ({ request }: LoaderFunctionArgs) => { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const url = new URL(request.url); const lookbackHours = parseInt(url.searchParams.get("lookbackHours") ?? "24", 10); diff --git a/apps/webapp/app/routes/admin.llm-models.new.tsx b/apps/webapp/app/routes/admin.llm-models.new.tsx index b4cd957b740..20c6e1461f2 100644 --- a/apps/webapp/app/routes/admin.llm-models.new.tsx +++ b/apps/webapp/app/routes/admin.llm-models.new.tsx @@ -14,7 +14,7 @@ import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; export const loader = async ({ request }: LoaderFunctionArgs) => { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); return typedjson({}); }; @@ -27,7 +27,7 @@ const CreateSchema = z.object({ export async function action({ request }: ActionFunctionArgs) { const userId = await requireUserId(request); const user = await prisma.user.findUnique({ where: { id: userId } }); - if (!user?.admin) return redirect("/"); + if (!user?.admin) throw redirect("/"); const formData = await request.formData(); const raw = Object.fromEntries(formData); @@ -57,7 +57,7 @@ export async function action({ request }: ActionFunctionArgs) { prices: Record; }>; try { - pricingTiers = JSON.parse(pricingTiersJson); + pricingTiers = JSON.parse(pricingTiersJson) as typeof pricingTiers; } catch { return typedjson({ error: "Invalid pricing tiers JSON" }, { status: 400 }); } diff --git a/apps/webapp/app/services/admin/missingLlmModels.server.ts b/apps/webapp/app/services/admin/missingLlmModels.server.ts index 0ad06e7e2d9..7ce6bc2ab7e 100644 --- a/apps/webapp/app/services/admin/missingLlmModels.server.ts +++ b/apps/webapp/app/services/admin/missingLlmModels.server.ts @@ -70,8 +70,9 @@ export async function getMissingLlmModels(opts: { // Filter out models that now have pricing in the database (added after spans were inserted). // The registry's match() handles prefix stripping for gateway/openrouter models. - if (!llmPricingRegistry?.isLoaded) return candidates; - return candidates.filter((c) => !llmPricingRegistry.match(c.model)); + if (!llmPricingRegistry || !llmPricingRegistry.isLoaded) return candidates; + const registry = llmPricingRegistry; + return candidates.filter((c) => !registry.match(c.model)); } export type MissingModelSample = { diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index a062e491de3..6b25bae2c5c 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -126,12 +126,12 @@ function enrichLlmCost(event: CreateEventInput): void { } event.style = { - ...event.style, + ...(event.style as Record | undefined), accessory: { style: "pills", items: pillItems, }, - }; + } as unknown as typeof event.style; // Only write llm_usage when cost data is available if (!cost && !providerCost) return; @@ -270,7 +270,7 @@ function extractProviderCost( let meta: Record; try { - meta = JSON.parse(rawMeta); + meta = JSON.parse(rawMeta) as Record; } catch { return null; } diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts index 9ffb14eba49..f6563521a25 100644 --- a/internal-packages/llm-pricing/src/registry.ts +++ b/internal-packages/llm-pricing/src/registry.ts @@ -1,4 +1,4 @@ -import type { PrismaClient } from "@trigger.dev/database"; +import type { PrismaClient, PrismaReplicaClient } from "@trigger.dev/database"; import type { LlmModelWithPricing, LlmCostResult, @@ -20,12 +20,12 @@ function compilePattern(pattern: string): RegExp { } export class ModelPricingRegistry { - private _prisma: PrismaClient; + private _prisma: PrismaClient | PrismaReplicaClient; private _patterns: CompiledPattern[] = []; private _exactMatchCache: Map = new Map(); private _loaded = false; - constructor(prisma: PrismaClient) { + constructor(prisma: PrismaClient | PrismaReplicaClient) { this._prisma = prisma; } diff --git a/internal-packages/tsql/src/query/schema.ts b/internal-packages/tsql/src/query/schema.ts index 2ea81091aad..00a28382de5 100644 --- a/internal-packages/tsql/src/query/schema.ts +++ b/internal-packages/tsql/src/query/schema.ts @@ -23,6 +23,9 @@ export type ClickHouseType = | "Date32" | "DateTime" | "DateTime64" + | "DateTime64(3)" + | "DateTime64(9)" + | "Decimal64(12)" | "UUID" | "Bool" | "JSON" @@ -281,6 +284,7 @@ export type ColumnFormatType = | "runId" | "runStatus" | "duration" + | "durationNs" | "durationSeconds" | "costInDollars" | "cost" From 2468dbc162f7c740cbdd2f7201af5ff8a346fa8d Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 13 Mar 2026 13:55:13 +0000 Subject: [PATCH 08/30] some fixes --- apps/webapp/test/otlpExporter.test.ts | 13 +++++++++++-- internal-packages/llm-pricing/package.json | 6 +++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/apps/webapp/test/otlpExporter.test.ts b/apps/webapp/test/otlpExporter.test.ts index 2a569b74d62..e7b047ac7dc 100644 --- a/apps/webapp/test/otlpExporter.test.ts +++ b/apps/webapp/test/otlpExporter.test.ts @@ -86,7 +86,7 @@ describe("OTLPExporter", () => { const event = $events[0]; expect(event.message).toBe("Responses API with 'gpt-4o'"); - expect(event.style).toEqual({ + expect(event.style).toMatchObject({ icon: "tabler-brand-openai", }); }); @@ -164,9 +164,18 @@ describe("OTLPExporter", () => { const event = $events[0]; expect(event.message).toBe("Responses API with gpt-4o"); - expect(event.style).toEqual({ + expect(event.style).toMatchObject({ icon: "tabler-brand-openai", }); + // Enrichment also adds model/token pills as accessories + const style = event.style as Record; + expect(style.accessory).toMatchObject({ + style: "pills", + items: expect.arrayContaining([ + expect.objectContaining({ text: "gpt-4o-2024-08-06" }), + expect.objectContaining({ text: "724" }), + ]), + }); }); it("should handle missing properties gracefully", () => { diff --git a/internal-packages/llm-pricing/package.json b/internal-packages/llm-pricing/package.json index 2ac51427dd6..8cf9e366f2c 100644 --- a/internal-packages/llm-pricing/package.json +++ b/internal-packages/llm-pricing/package.json @@ -11,8 +11,8 @@ }, "scripts": { "typecheck": "tsc --noEmit", - "generate": "./scripts/sync-model-prices.sh", - "sync-prices": "./scripts/sync-model-prices.sh", - "sync-prices:check": "./scripts/sync-model-prices.sh --check" + "generate": "echo 'defaultPrices.ts is pre-committed — run sync-prices to update'", + "sync-prices": "bash scripts/sync-model-prices.sh", + "sync-prices:check": "bash scripts/sync-model-prices.sh --check" } } From 9bceee33427533790bc74c0d8f90dfeea5432cfe Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Fri, 13 Mar 2026 14:23:37 +0000 Subject: [PATCH 09/30] A bunch of fixes --- .../components/runs/v3/ai/AISpanDetails.tsx | 21 +-- .../app/routes/admin.llm-models.$modelId.tsx | 14 +- apps/webapp/app/v3/querySchemas.ts | 1 - .../v3/utils/enrichCreatableEvents.server.ts | 4 +- apps/webapp/test/otlpExporter.test.ts | 64 +++++++ .../llm-pricing/src/registry.test.ts | 159 ++++++++++++++++++ internal-packages/llm-pricing/src/registry.ts | 18 +- 7 files changed, 255 insertions(+), 26 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx index 988de788167..73dfa6d5bc6 100644 --- a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx @@ -16,8 +16,7 @@ export function AISpanDetails({ rawProperties?: string; }) { const [tab, setTab] = useState("overview"); - const hasTools = - (aiData.toolDefinitions && aiData.toolDefinitions.length > 0) || aiData.toolCount != null; + const toolCount = aiData.toolCount ?? aiData.toolDefinitions?.length ?? 0; return (
@@ -40,16 +39,14 @@ export function AISpanDetails({ > Messages - {hasTools && ( - setTab("tools")} - shortcut={{ key: "t" }} - > - Tools{aiData.toolCount != null ? ` (${aiData.toolCount})` : ""} - - )} + setTab("tools")} + shortcut={{ key: "t" }} + > + Tools{toolCount > 0 ? ` (${toolCount})` : ""} +
diff --git a/apps/webapp/app/routes/admin.llm-models.$modelId.tsx b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx index 7f7fadaf436..e37491a1b4f 100644 --- a/apps/webapp/app/routes/admin.llm-models.$modelId.tsx +++ b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx @@ -1,4 +1,4 @@ -import { Form, useNavigate } from "@remix-run/react"; +import { Form, useActionData, useNavigate } from "@remix-run/react"; import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; import { redirect } from "@remix-run/server-runtime"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; @@ -130,6 +130,7 @@ export async function action({ request, params }: ActionFunctionArgs) { export default function AdminLlmModelDetailRoute() { const { model } = useTypedLoaderData(); + const actionData = useActionData<{ success?: boolean; error?: string; details?: unknown[] }>(); const navigate = useNavigate(); const [modelName, setModelName] = useState(model.modelName); @@ -273,6 +274,17 @@ export default function AdminLlmModelDetailRoute() { ))}
+ {actionData?.error && ( +
+ {actionData.error} + {actionData.details && ( + 
+                    {JSON.stringify(actionData.details, null, 2)}
+
+ )} +
+ )} + {/* Actions */}
{/* Tab content */} -
+
{tab === "overview" && } {tab === "messages" && } {tab === "tools" && }
- {/* Footer: Copy raw */} - {rawProperties && } + {/* Footer: Copy raw (admin only) */} + {isAdmin && rawProperties && }
); } @@ -126,23 +129,15 @@ function CopyRawFooter({ rawProperties }: { rawProperties: string }) { } return ( -
- + Copy raw properties +
); } From 83c4190698a6cfd5070de49a65c790a0d674f8cf Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Fri, 13 Mar 2026 16:25:04 +0000 Subject: [PATCH 11/30] Nicer Tools tab icon and tabs scroll behaviour --- .../app/components/runs/v3/ai/AISpanDetails.tsx | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx index b057411e202..917062acad3 100644 --- a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx @@ -24,7 +24,7 @@ export function AISpanDetails({ return (
{/* Tab bar */} -
+
setTab("tools")} shortcut={{ key: "t" }} > - Tools{toolCount > 0 ? ` (${toolCount})` : ""} + + Tools + {toolCount > 0 && ( + + {toolCount} + + )} +
From afdd619fcb1507797517008b2406bc01c4d3bd4a Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Fri, 13 Mar 2026 17:07:17 +0000 Subject: [PATCH 12/30] Display details as a property table instead of pills --- .../components/runs/v3/ai/AIChatMessages.tsx | 3 +- .../components/runs/v3/ai/AIModelSummary.tsx | 106 +++++++----------- .../components/runs/v3/ai/AISpanDetails.tsx | 11 +- 3 files changed, 48 insertions(+), 72 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx index 9fe39482c86..5c783258228 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx @@ -1,5 +1,6 @@ import { lazy, Suspense, useState } from "react"; import { CodeBlock } from "~/components/code/CodeBlock"; +import { Header3 } from "~/components/primitives/Headers"; import type { DisplayItem, ToolUse } from "./types"; // Lazy load streamdown to avoid SSR issues @@ -45,7 +46,7 @@ function SectionHeader({ }) { return (
- {label} + {label} {right &&
{right}
}
); diff --git a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx index a33b90ef899..62341fc9041 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIModelSummary.tsx @@ -1,33 +1,36 @@ import { formatCurrencyAccurate } from "~/utils/numberFormatter"; +import { Header3 } from "~/components/primitives/Headers"; import type { AISpanData } from "./types"; export function AITagsRow({ aiData }: { aiData: AISpanData }) { return ( -
- {aiData.model} - {aiData.provider !== "unknown" && {aiData.provider}} - {aiData.resolvedProvider && ( - via {aiData.resolvedProvider} - )} - {aiData.finishReason && {aiData.finishReason}} - {aiData.serviceTier && tier: {aiData.serviceTier}} - {aiData.toolChoice && tools: {aiData.toolChoice}} - {aiData.toolCount != null && aiData.toolCount > 0 && ( - - {aiData.toolCount} {aiData.toolCount === 1 ? "tool" : "tools"} - - )} - {aiData.messageCount != null && ( - - {aiData.messageCount} {aiData.messageCount === 1 ? "msg" : "msgs"} - - )} - {aiData.telemetryMetadata && - Object.entries(aiData.telemetryMetadata).map(([key, value]) => ( - - {key}: {value} - - ))} +
+
+ + {aiData.provider !== "unknown" && } + {aiData.resolvedProvider && ( + + )} + {aiData.finishReason && } + {aiData.serviceTier && } + {aiData.toolChoice && } + {aiData.toolCount != null && aiData.toolCount > 0 && ( + + )} + {aiData.messageCount != null && ( + + )} + {aiData.telemetryMetadata && + Object.entries(aiData.telemetryMetadata).map(([key, value]) => ( + + ))} +
); } @@ -35,13 +38,16 @@ export function AITagsRow({ aiData }: { aiData: AISpanData }) { export function AIStatsSummary({ aiData }: { aiData: AISpanData }) { return (
- Stats - -
+ Stats +
{aiData.cachedTokens != null && aiData.cachedTokens > 0 && ( - + )} {aiData.cacheCreationTokens != null && aiData.cacheCreationTokens > 0 && ( )} - + {aiData.totalCost != null && ( @@ -84,22 +84,20 @@ function MetricRow({ value, unit, bold, - border, }: { label: string; value: string; unit?: string; bold?: boolean; - border?: boolean; }) { return ( -
+
{label} - + {value} {unit && {unit}} @@ -113,23 +111,3 @@ function formatTtfc(ms: number): string { } return `${Math.round(ms)}ms`; } - -function Pill({ - children, - variant = "default", -}: { - children: React.ReactNode; - variant?: "default" | "dimmed"; -}) { - return ( - - {children} - - ); -} diff --git a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx index 917062acad3..1c8a64fc0f8 100644 --- a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx @@ -1,6 +1,7 @@ import { CheckIcon, ClipboardDocumentIcon } from "@heroicons/react/20/solid"; import { useState } from "react"; import { Button } from "~/components/primitives/Buttons"; +import { Header3 } from "~/components/primitives/Headers"; import { TabButton, TabContainer } from "~/components/primitives/Tabs"; import { useHasAdminAccess } from "~/hooks/useUser"; import { AIChatMessages, AssistantResponse } from "./AIChatMessages"; @@ -77,7 +78,7 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) { const { userText, outputText, outputToolNames } = extractInputOutput(aiData); return ( -
+
{/* Tags + Stats */} @@ -85,9 +86,7 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) { {/* Input (last user prompt) */} {userText && (
- - Input - + Input

{userText}

)} @@ -96,9 +95,7 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) { {outputText && } {outputToolNames.length > 0 && !outputText && (
- - Output - + Output

Called {outputToolNames.length === 1 ? "tool" : "tools"}:{" "} {outputToolNames.join(", ")} From 46796451846a2891a1794dd2d07296370ecbe3e3 Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Fri, 13 Mar 2026 17:11:53 +0000 Subject: [PATCH 13/30] Use proper paragraph component --- apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx b/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx index 5130256f85b..a329698dd5e 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx @@ -1,6 +1,7 @@ import { useState } from "react"; import { CodeBlock } from "~/components/code/CodeBlock"; import type { AISpanData, ToolDefinition } from "./types"; +import { Paragraph } from "~/components/primitives/Paragraph"; export function AIToolsInventory({ aiData }: { aiData: AISpanData }) { const defs = aiData.toolDefinitions ?? []; @@ -8,8 +9,8 @@ export function AIToolsInventory({ aiData }: { aiData: AISpanData }) { if (defs.length === 0) { return ( -

- No tool definitions available for this span. +
+ No tool definitions available for this span.
); } From aa83167abc55c70220304b7945b421504306c959 Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Fri, 13 Mar 2026 17:47:13 +0000 Subject: [PATCH 14/30] Style improvements to the messages --- .../components/runs/v3/ai/AIChatMessages.tsx | 99 ++++++++++++------- .../components/runs/v3/ai/AISpanDetails.tsx | 23 +++-- 2 files changed, 79 insertions(+), 43 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx index 5c783258228..a50ef8ae806 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx @@ -1,6 +1,9 @@ +import { CheckIcon, ClipboardDocumentIcon, CodeBracketSquareIcon } from "@heroicons/react/20/solid"; import { lazy, Suspense, useState } from "react"; import { CodeBlock } from "~/components/code/CodeBlock"; +import { Button } from "~/components/primitives/Buttons"; import { Header3 } from "~/components/primitives/Headers"; +import { Paragraph } from "~/components/primitives/Paragraph"; import type { DisplayItem, ToolUse } from "./types"; // Lazy load streamdown to avoid SSR issues @@ -16,7 +19,7 @@ const StreamdownRenderer = lazy(() => export function AIChatMessages({ items }: { items: DisplayItem[] }) { return ( -
+
{items.map((item, i) => { switch (item.type) { case "system": @@ -37,13 +40,7 @@ export function AIChatMessages({ items }: { items: DisplayItem[] }) { // Section header (shared across all sections) // --------------------------------------------------------------------------- -function SectionHeader({ - label, - right, -}: { - label: string; - right?: React.ReactNode; -}) { +function SectionHeader({ label, right }: { label: string; right?: React.ReactNode }) { return (
{label} @@ -52,6 +49,14 @@ function SectionHeader({ ); } +export function ChatBubble({ children }: { children: React.ReactNode }) { + return ( +
+ {children} +
+ ); +} + // --------------------------------------------------------------------------- // System // --------------------------------------------------------------------------- @@ -62,7 +67,7 @@ function SystemSection({ text }: { text: string }) { const preview = isLong ? text.slice(0, 150) + "..." : text; return ( -
+
- 
-        {expanded || !isLong ? text : preview}
-
+ + + {expanded || !isLong ? text : preview} + +
); } @@ -89,9 +96,11 @@ function SystemSection({ text }: { text: string }) { function UserSection({ text }: { text: string }) { return ( -
+
-

{text}

+ + {text} +
); } @@ -108,36 +117,54 @@ export function AssistantResponse({ headerLabel?: string; }) { const [mode, setMode] = useState<"rendered" | "raw">("rendered"); + const [copied, setCopied] = useState(false); + + function handleCopy() { + navigator.clipboard.writeText(text); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } return ( -
+
- - + +
} /> {mode === "rendered" ? ( -
- {text}}> - {text} - -
+ + + {text}}> + {text} + + + ) : ( - + )}
); @@ -151,9 +178,13 @@ function ToolUseSection({ tools }: { tools: ToolUse[] }) { return (
- {tools.map((tool) => ( - - ))} + +
+ {tools.map((tool) => ( + + ))} +
+
); } @@ -228,9 +259,9 @@ function ToolUseRow({ tool }: { tool: ToolUse }) { )} {activeTab === "details" && hasDetails && ( -
+
{tool.description && ( -

{tool.description}

+

{tool.description}

)} {tool.parametersJson && (
diff --git a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx index 1c8a64fc0f8..4b64da7db38 100644 --- a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx @@ -2,9 +2,10 @@ import { CheckIcon, ClipboardDocumentIcon } from "@heroicons/react/20/solid"; import { useState } from "react"; import { Button } from "~/components/primitives/Buttons"; import { Header3 } from "~/components/primitives/Headers"; +import { Paragraph } from "~/components/primitives/Paragraph"; import { TabButton, TabContainer } from "~/components/primitives/Tabs"; import { useHasAdminAccess } from "~/hooks/useUser"; -import { AIChatMessages, AssistantResponse } from "./AIChatMessages"; +import { AIChatMessages, AssistantResponse, ChatBubble } from "./AIChatMessages"; import { AIStatsSummary, AITagsRow } from "./AIModelSummary"; import { AIToolsInventory } from "./AIToolsInventory"; import type { AISpanData, DisplayItem } from "./types"; @@ -85,21 +86,25 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) { {/* Input (last user prompt) */} {userText && ( -
+
Input -

{userText}

+ + {userText} +
)} {/* Output (assistant response or tool calls) */} {outputText && } {outputToolNames.length > 0 && !outputText && ( -
+
Output -

- Called {outputToolNames.length === 1 ? "tool" : "tools"}:{" "} - {outputToolNames.join(", ")} -

+ + + Called {outputToolNames.length === 1 ? "tool" : "tools"}:{" "} + {outputToolNames.join(", ")} + +
)}
@@ -109,7 +114,7 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) { function MessagesTab({ aiData }: { aiData: AISpanData }) { return (
-
+
{aiData.items && aiData.items.length > 0 && } {aiData.responseText && !hasAssistantItem(aiData.items) && ( From e9f13b9fb011ebe8a99650c8936252244c917e15 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Sat, 14 Mar 2026 09:55:26 +0000 Subject: [PATCH 15/30] Renamed llm_usage_v1 to llm_metrics_v1, added some additional fields to track as well --- .server-changes/llm-cost-tracking.md | 2 +- .../ExamplesContent.tsx | 24 +++---- .../route.tsx | 6 ++ .../clickhouseEventRepository.server.ts | 67 ++++++++++--------- .../eventRepository/eventRepository.types.ts | 11 ++- apps/webapp/app/v3/querySchemas.ts | 61 +++++++++++++---- .../v3/utils/enrichCreatableEvents.server.ts | 39 ++++++++--- apps/webapp/seed-ai-spans.mts | 23 ++++--- apps/webapp/seed.mts | 60 +++++++++++++++++ apps/webapp/test/otlpExporter.test.ts | 30 ++++----- .../schema/024_create_llm_usage_v1.sql | 22 +++++- internal-packages/clickhouse/src/index.ts | 8 +-- .../src/{llmUsage.ts => llmMetrics.ts} | 21 ++++-- 13 files changed, 269 insertions(+), 105 deletions(-) rename internal-packages/clickhouse/src/{llmUsage.ts => llmMetrics.ts} (64%) diff --git a/.server-changes/llm-cost-tracking.md b/.server-changes/llm-cost-tracking.md index b68302731a7..7567aae7d1b 100644 --- a/.server-changes/llm-cost-tracking.md +++ b/.server-changes/llm-cost-tracking.md @@ -3,4 +3,4 @@ area: webapp type: feature --- -Add automatic LLM cost calculation for spans with GenAI semantic conventions. When a span arrives with `gen_ai.response.model` and token usage data, costs are calculated from an in-memory pricing registry backed by Postgres and dual-written to both span attributes (`trigger.llm.*`) and a new `llm_usage_v1` ClickHouse table. +Add automatic LLM cost calculation for spans with GenAI semantic conventions. When a span arrives with `gen_ai.response.model` and token usage data, costs are calculated from an in-memory pricing registry backed by Postgres and dual-written to both span attributes (`trigger.llm.*`) and a new `llm_metrics_v1` ClickHouse table that captures usage, cost, performance (TTFC, tokens/sec), and behavioral (finish reason, operation type) metrics. diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx index 1235b443348..f0c3c1d616f 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx @@ -126,12 +126,12 @@ LIMIT 100`, SUM(total_cost) AS total_cost, SUM(input_tokens) AS input_tokens, SUM(output_tokens) AS output_tokens -FROM llm_usage +FROM llm_metrics WHERE start_time > now() - INTERVAL 7 DAY GROUP BY response_model ORDER BY total_cost DESC`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, { title: "LLM cost over time", @@ -139,12 +139,12 @@ ORDER BY total_cost DESC`, query: `SELECT timeBucket(), SUM(total_cost) AS total_cost -FROM llm_usage +FROM llm_metrics GROUP BY timeBucket ORDER BY timeBucket LIMIT 1000`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, { title: "Most expensive runs by LLM cost (top 50)", @@ -155,12 +155,12 @@ LIMIT 1000`, SUM(total_cost) AS llm_cost, SUM(input_tokens) AS input_tokens, SUM(output_tokens) AS output_tokens -FROM llm_usage +FROM llm_metrics GROUP BY run_id, task_identifier ORDER BY llm_cost DESC LIMIT 50`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, { title: "LLM calls by provider", @@ -169,11 +169,11 @@ LIMIT 50`, gen_ai_system, count() AS call_count, SUM(total_cost) AS total_cost -FROM llm_usage +FROM llm_metrics GROUP BY gen_ai_system ORDER BY total_cost DESC`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, { title: "LLM cost by user", @@ -184,13 +184,13 @@ ORDER BY total_cost DESC`, SUM(total_cost) AS total_cost, SUM(total_tokens) AS total_tokens, count() AS call_count -FROM llm_usage +FROM llm_metrics WHERE metadata.userId != '' GROUP BY metadata.userId ORDER BY total_cost DESC LIMIT 50`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, { title: "LLM cost by metadata key", @@ -202,11 +202,11 @@ LIMIT 50`, total_cost, total_tokens, run_id -FROM llm_usage +FROM llm_metrics ORDER BY start_time DESC LIMIT 20`, scope: "environment", - table: "llm_usage", + table: "llm_metrics", }, ]; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx index 29d9e246e2d..ee69419e1b7 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx @@ -120,6 +120,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { return typedjson({ ...result, regions: regionsResult.regions }); } catch (error) { + logger.error("Failed to load test page", { + taskParam, + error: error instanceof Error ? error.message : error, + stack: error instanceof Error ? error.stack : undefined, + }); + return redirectWithErrorMessage( v3TestPath({ slug: organizationSlug }, { slug: projectParam }, environment), request, diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts index 531c3e307e9..86f3560669c 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts @@ -1,6 +1,6 @@ import type { ClickHouse, - LlmUsageV1Input, + LlmMetricsV1Input, TaskEventDetailedSummaryV1Result, TaskEventDetailsV1Result, TaskEventSummaryV1Result, @@ -96,7 +96,7 @@ export class ClickhouseEventRepository implements IEventRepository { private _clickhouse: ClickHouse; private _config: ClickhouseEventRepositoryConfig; private readonly _flushScheduler: DynamicFlushScheduler; - private readonly _llmUsageFlushScheduler: DynamicFlushScheduler; + private readonly _llmMetricsFlushScheduler: DynamicFlushScheduler; private _tracer: Tracer; private _version: "v1" | "v2"; @@ -122,10 +122,10 @@ export class ClickhouseEventRepository implements IEventRepository { }, }); - this._llmUsageFlushScheduler = new DynamicFlushScheduler({ + this._llmMetricsFlushScheduler = new DynamicFlushScheduler({ batchSize: 5000, flushInterval: 2000, - callback: this.#flushLlmUsageBatch.bind(this), + callback: this.#flushLlmMetricsBatch.bind(this), minConcurrency: 1, maxConcurrency: 2, maxBatchSize: 10000, @@ -230,9 +230,9 @@ export class ClickhouseEventRepository implements IEventRepository { }); } - async #flushLlmUsageBatch(flushId: string, rows: LlmUsageV1Input[]) { + async #flushLlmMetricsBatch(flushId: string, rows: LlmMetricsV1Input[]) { - const [insertError] = await this._clickhouse.llmUsage.insert(rows, { + const [insertError] = await this._clickhouse.llmMetrics.insert(rows, { params: { clickhouse_settings: this.#getClickhouseInsertSettings(), }, @@ -242,13 +242,13 @@ export class ClickhouseEventRepository implements IEventRepository { throw insertError; } - logger.info("ClickhouseEventRepository.flushLlmUsageBatch Inserted LLM usage batch", { + logger.info("ClickhouseEventRepository.flushLlmMetricsBatch Inserted LLM metrics batch", { rows: rows.length, }); } - #createLlmUsageInput(event: CreateEventInput): LlmUsageV1Input { - const llmUsage = event._llmUsage!; + #createLlmMetricsInput(event: CreateEventInput): LlmMetricsV1Input { + const llmMetrics = event._llmMetrics!; return { organization_id: event.organizationId, @@ -258,22 +258,27 @@ export class ClickhouseEventRepository implements IEventRepository { task_identifier: event.taskSlug, trace_id: event.traceId, span_id: event.spanId, - gen_ai_system: llmUsage.genAiSystem, - request_model: llmUsage.requestModel, - response_model: llmUsage.responseModel, - matched_model_id: llmUsage.matchedModelId, - operation_name: llmUsage.operationName, - pricing_tier_id: llmUsage.pricingTierId, - pricing_tier_name: llmUsage.pricingTierName, - input_tokens: llmUsage.inputTokens, - output_tokens: llmUsage.outputTokens, - total_tokens: llmUsage.totalTokens, - usage_details: llmUsage.usageDetails, - input_cost: llmUsage.inputCost, - output_cost: llmUsage.outputCost, - total_cost: llmUsage.totalCost, - cost_details: llmUsage.costDetails, - metadata: llmUsage.metadata, + gen_ai_system: llmMetrics.genAiSystem, + request_model: llmMetrics.requestModel, + response_model: llmMetrics.responseModel, + matched_model_id: llmMetrics.matchedModelId, + operation_id: llmMetrics.operationId, + finish_reason: llmMetrics.finishReason, + cost_source: llmMetrics.costSource, + pricing_tier_id: llmMetrics.pricingTierId, + pricing_tier_name: llmMetrics.pricingTierName, + input_tokens: llmMetrics.inputTokens, + output_tokens: llmMetrics.outputTokens, + total_tokens: llmMetrics.totalTokens, + usage_details: llmMetrics.usageDetails, + input_cost: llmMetrics.inputCost, + output_cost: llmMetrics.outputCost, + total_cost: llmMetrics.totalCost, + cost_details: llmMetrics.costDetails, + provider_cost: llmMetrics.providerCost, + ms_to_first_chunk: llmMetrics.msToFirstChunk, + tokens_per_second: llmMetrics.tokensPerSecond, + metadata: llmMetrics.metadata, start_time: this.#clampAndFormatStartTime(event.startTime.toString()), duration: formatClickhouseUnsignedIntegerString(event.duration ?? 0), }; @@ -300,13 +305,13 @@ export class ClickhouseEventRepository implements IEventRepository { async insertMany(events: CreateEventInput[]): Promise { this.addToBatch(events.flatMap((event) => this.createEventToTaskEventV1Input(event))); - // Dual-write LLM usage records for spans with cost enrichment - const llmUsageRows = events - .filter((e) => e._llmUsage != null) - .map((e) => this.#createLlmUsageInput(e)); + // Dual-write LLM metrics records for spans with cost enrichment + const llmMetricsRows = events + .filter((e) => e._llmMetrics != null) + .map((e) => this.#createLlmMetricsInput(e)); - if (llmUsageRows.length > 0) { - this._llmUsageFlushScheduler.addToBatch(llmUsageRows); + if (llmMetricsRows.length > 0) { + this._llmMetricsFlushScheduler.addToBatch(llmMetricsRows); } } diff --git a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts index 69590dc9493..fcef0010ee0 100644 --- a/apps/webapp/app/v3/eventRepository/eventRepository.types.ts +++ b/apps/webapp/app/v3/eventRepository/eventRepository.types.ts @@ -21,12 +21,14 @@ export type { ExceptionEventProperties }; // Event Creation Types // ============================================================================ -export type LlmUsageData = { +export type LlmMetricsData = { genAiSystem: string; requestModel: string; responseModel: string; matchedModelId: string; - operationName: string; + operationId: string; + finishReason: string; + costSource: string; pricingTierId: string; pricingTierName: string; inputTokens: number; @@ -37,6 +39,9 @@ export type LlmUsageData = { outputCost: number; totalCost: number; costDetails: Record; + providerCost: number; + msToFirstChunk: number; + tokensPerSecond: number; metadata: Record; }; @@ -78,7 +83,7 @@ export type CreateEventInput = Omit< machineId?: string; runTags?: string[]; /** Side-channel data for LLM cost tracking, populated by enrichCreatableEvents */ - _llmUsage?: LlmUsageData; + _llmMetrics?: LlmMetricsData; }; export type CreatableEventKind = TaskEventKind; diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index 7ec23285d0c..b4f4fceb80c 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -600,12 +600,12 @@ export const metricsSchema: TableSchema = { * All available schemas for the query editor */ /** - * Schema definition for the llm_usage table (trigger_dev.llm_usage_v1) + * Schema definition for the llm_metrics table (trigger_dev.llm_metrics_v1) */ -export const llmUsageSchema: TableSchema = { - name: "llm_usage", - clickhouseName: "trigger_dev.llm_usage_v1", - description: "LLM token usage and cost data from GenAI spans", +export const llmMetricsSchema: TableSchema = { + name: "llm_metrics", + clickhouseName: "trigger_dev.llm_metrics_v1", + description: "LLM metrics: token usage, cost, performance, and behavior from GenAI spans", timeConstraint: "start_time", tenantColumns: { organizationId: "organization_id", @@ -669,11 +669,26 @@ export const llmUsageSchema: TableSchema = { coreColumn: true, }), }, - operation_name: { - name: "operation_name", + operation_id: { + name: "operation_id", ...column("LowCardinality(String)", { - description: "Operation type (e.g. chat, completion)", - example: "chat", + description: "Operation type (e.g. ai.streamText.doStream, ai.generateText.doGenerate)", + example: "ai.streamText.doStream", + }), + }, + finish_reason: { + name: "finish_reason", + ...column("LowCardinality(String)", { + description: "Why the LLM stopped generating (e.g. stop, tool-calls, length)", + example: "stop", + coreColumn: true, + }), + }, + cost_source: { + name: "cost_source", + ...column("LowCardinality(String)", { + description: "Where cost data came from (registry, gateway, openrouter)", + example: "registry", }), }, input_tokens: { @@ -700,14 +715,14 @@ export const llmUsageSchema: TableSchema = { input_cost: { name: "input_cost", ...column("Decimal64(12)", { - description: "Input cost in USD", + description: "Input cost in USD (from pricing registry)", customRenderType: "costInDollars", }), }, output_cost: { name: "output_cost", ...column("Decimal64(12)", { - description: "Output cost in USD", + description: "Output cost in USD (from pricing registry)", customRenderType: "costInDollars", }), }, @@ -719,6 +734,28 @@ export const llmUsageSchema: TableSchema = { coreColumn: true, }), }, + provider_cost: { + name: "provider_cost", + ...column("Decimal64(12)", { + description: "Provider-reported cost in USD (from gateway or openrouter)", + customRenderType: "costInDollars", + }), + }, + ms_to_first_chunk: { + name: "ms_to_first_chunk", + ...column("Float64", { + description: "Time to first chunk in milliseconds (TTFC)", + example: "245.3", + coreColumn: true, + }), + }, + tokens_per_second: { + name: "tokens_per_second", + ...column("Float64", { + description: "Average output tokens per second", + example: "72.5", + }), + }, pricing_tier_name: { name: "pricing_tier_name", ...column("LowCardinality(String)", { @@ -751,7 +788,7 @@ export const llmUsageSchema: TableSchema = { }, }; -export const querySchemas: TableSchema[] = [runsSchema, metricsSchema, llmUsageSchema]; +export const querySchemas: TableSchema[] = [runsSchema, metricsSchema, llmMetricsSchema]; /** * Default query for the query editor diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index 29be7ae6fad..b9ed86aa874 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -1,4 +1,4 @@ -import type { CreateEventInput, LlmUsageData } from "../eventRepository/eventRepository.types"; +import type { CreateEventInput, LlmMetricsData } from "../eventRepository/eventRepository.types"; // Registry interface — matches ModelPricingRegistry from @internal/llm-pricing type CostRegistry = { @@ -36,12 +36,12 @@ function enrichCreatableEvent(event: CreateEventInput): CreateEventInput { event.message = message; event.style = enrichStyle(event); - enrichLlmCost(event); + enrichLlmMetrics(event); return event; } -function enrichLlmCost(event: CreateEventInput): void { +function enrichLlmMetrics(event: CreateEventInput): void { const props = event.properties; if (!props) return; @@ -133,7 +133,7 @@ function enrichLlmCost(event: CreateEventInput): void { }, } as unknown as typeof event.style; - // Only write llm_usage when cost data is available + // Only write llm_metrics when cost data is available if (!cost && !providerCost) return; // Build metadata map from run tags and ai.telemetry.metadata.* @@ -154,13 +154,33 @@ function enrichLlmCost(event: CreateEventInput): void { } } - // Set _llmUsage side-channel for dual-write to llm_usage_v1 - const llmUsage: LlmUsageData = { + // Extract new performance/behavioral fields + const finishReason = typeof props["ai.response.finishReason"] === "string" + ? props["ai.response.finishReason"] + : typeof props["gen_ai.response.finish_reasons"] === "string" + ? props["gen_ai.response.finish_reasons"] + : ""; + const operationId = typeof props["ai.operationId"] === "string" + ? props["ai.operationId"] + : (props["gen_ai.operation.name"] as string) ?? (props["operation.name"] as string) ?? ""; + const msToFirstChunk = typeof props["ai.response.msToFirstChunk"] === "number" + ? props["ai.response.msToFirstChunk"] + : 0; + const avgTokensPerSec = typeof props["ai.response.avgOutputTokensPerSecond"] === "number" + ? props["ai.response.avgOutputTokensPerSecond"] + : 0; + const costSource = cost ? "registry" : providerCost ? providerCost.source : ""; + const providerCostValue = providerCost?.totalCost ?? 0; + + // Set _llmMetrics side-channel for dual-write to llm_metrics_v1 + const llmMetrics: LlmMetricsData = { genAiSystem: (props["gen_ai.system"] as string) ?? "unknown", requestModel: (props["gen_ai.request.model"] as string) ?? responseModel, responseModel, matchedModelId: cost?.matchedModelId ?? "", - operationName: (props["gen_ai.operation.name"] as string) ?? (props["operation.name"] as string) ?? "", + operationId, + finishReason, + costSource, pricingTierId: cost?.pricingTierId ?? (providerCost ? `provider:${providerCost.source}` : ""), pricingTierName: cost?.pricingTierName ?? (providerCost ? `${providerCost.source} reported` : ""), inputTokens: usageDetails["input"] ?? 0, @@ -171,10 +191,13 @@ function enrichLlmCost(event: CreateEventInput): void { outputCost: cost?.outputCost ?? 0, totalCost: cost?.totalCost ?? providerCost?.totalCost ?? 0, costDetails: cost?.costDetails ?? {}, + providerCost: providerCostValue, + msToFirstChunk, + tokensPerSecond: avgTokensPerSec, metadata, }; - event._llmUsage = llmUsage; + event._llmMetrics = llmMetrics; } function extractUsageDetails(props: Record): Record { diff --git a/apps/webapp/seed-ai-spans.mts b/apps/webapp/seed-ai-spans.mts index 19a72c23f22..ef6a8f4b6ec 100644 --- a/apps/webapp/seed-ai-spans.mts +++ b/apps/webapp/seed-ai-spans.mts @@ -4,7 +4,7 @@ import { prisma } from "./app/db.server"; import { createOrganization } from "./app/models/organization.server"; import { createProject } from "./app/models/project.server"; import { ClickHouse } from "@internal/clickhouse"; -import type { TaskEventV2Input, LlmUsageV1Input } from "@internal/clickhouse"; +import type { TaskEventV2Input, LlmMetricsV1Input } from "@internal/clickhouse"; import { generateTraceId, generateSpanId, @@ -117,8 +117,8 @@ function eventToClickhouseRow(event: CreateEventInput): TaskEventV2Input { }; } -function eventToLlmUsageRow(event: CreateEventInput): LlmUsageV1Input { - const llm = event._llmUsage!; +function eventToLlmMetricsRow(event: CreateEventInput): LlmMetricsV1Input { + const llm = event._llmMetrics!; return { organization_id: event.organizationId, project_id: event.projectId, @@ -131,7 +131,9 @@ function eventToLlmUsageRow(event: CreateEventInput): LlmUsageV1Input { request_model: llm.requestModel, response_model: llm.responseModel, matched_model_id: llm.matchedModelId, - operation_name: llm.operationName, + operation_id: llm.operationId, + finish_reason: llm.finishReason, + cost_source: llm.costSource, pricing_tier_id: llm.pricingTierId, pricing_tier_name: llm.pricingTierName, input_tokens: llm.inputTokens, @@ -142,6 +144,9 @@ function eventToLlmUsageRow(event: CreateEventInput): LlmUsageV1Input { output_cost: llm.outputCost, total_cost: llm.totalCost, cost_details: llm.costDetails, + provider_cost: llm.providerCost, + ms_to_first_chunk: llm.msToFirstChunk, + tokens_per_second: llm.tokensPerSecond, metadata: llm.metadata, start_time: formatStartTime(BigInt(event.startTime)), duration: formatDuration(event.duration ?? 0), @@ -335,8 +340,8 @@ async function seedAiSpans() { crumb("enriching events"); // @crumbs const enriched = enrichCreatableEvents(events); - const enrichedCount = enriched.filter((e) => e._llmUsage != null).length; - const totalCost = enriched.reduce((sum, e) => sum + (e._llmUsage?.totalCost ?? 0), 0); + const enrichedCount = enriched.filter((e) => e._llmMetrics != null).length; + const totalCost = enriched.reduce((sum, e) => sum + (e._llmMetrics?.totalCost ?? 0), 0); console.log( `Enriched ${enrichedCount} spans with LLM cost (total: $${totalCost.toFixed(6)})` ); @@ -362,10 +367,10 @@ async function seedAiSpans() { crumb("task events inserted", { rowCount: chRows.length }); // @crumbs // Insert LLM usage rows - const llmRows = enriched.filter((e) => e._llmUsage != null).map(eventToLlmUsageRow); + const llmRows = enriched.filter((e) => e._llmMetrics != null).map(eventToLlmMetricsRow); if (llmRows.length > 0) { - await clickhouse.llmUsage.insert(llmRows); - crumb("llm usage inserted", { rowCount: llmRows.length }); // @crumbs + await clickhouse.llmMetrics.insert(llmRows); + crumb("llm metrics inserted", { rowCount: llmRows.length }); // @crumbs } // 12. Output diff --git a/apps/webapp/seed.mts b/apps/webapp/seed.mts index aa08eaaeec0..9eb30cd2503 100644 --- a/apps/webapp/seed.mts +++ b/apps/webapp/seed.mts @@ -75,6 +75,7 @@ async function seed() { } await createBatchLimitOrgs(user); + await ensureDefaultWorkerGroup(); console.log("\n🎉 Seed complete!\n"); console.log("Summary:"); @@ -249,3 +250,62 @@ async function findOrCreateProject( return { project, environments }; } + +async function ensureDefaultWorkerGroup() { + // Check if the feature flag already exists + const existingFlag = await prisma.featureFlag.findUnique({ + where: { key: "defaultWorkerInstanceGroupId" }, + }); + + if (existingFlag) { + console.log(`✅ Default worker instance group already configured`); + return; + } + + // Check if a managed worker group already exists + let workerGroup = await prisma.workerInstanceGroup.findFirst({ + where: { type: "MANAGED" }, + }); + + if (!workerGroup) { + console.log("Creating default worker instance group..."); + + const { createHash, randomBytes } = await import("crypto"); + const tokenValue = `tr_wgt_${randomBytes(20).toString("hex")}`; + const tokenHash = createHash("sha256").update(tokenValue).digest("hex"); + + const token = await prisma.workerGroupToken.create({ + data: { tokenHash }, + }); + + workerGroup = await prisma.workerInstanceGroup.create({ + data: { + type: "MANAGED", + name: "local-dev", + masterQueue: "local-dev", + description: "Local development worker group", + tokenId: token.id, + }, + }); + + console.log(`✅ Created worker instance group: ${workerGroup.name} (${workerGroup.id})`); + } else { + console.log( + `✅ Worker instance group already exists: ${workerGroup.name} (${workerGroup.id})` + ); + } + + // Set the feature flag + await prisma.featureFlag.upsert({ + where: { key: "defaultWorkerInstanceGroupId" }, + create: { + key: "defaultWorkerInstanceGroupId", + value: workerGroup.id, + }, + update: { + value: workerGroup.id, + }, + }); + + console.log(`✅ Set defaultWorkerInstanceGroupId feature flag`); +} diff --git a/apps/webapp/test/otlpExporter.test.ts b/apps/webapp/test/otlpExporter.test.ts index 6ba54243faf..0194ae6bf74 100644 --- a/apps/webapp/test/otlpExporter.test.ts +++ b/apps/webapp/test/otlpExporter.test.ts @@ -505,20 +505,20 @@ describe("OTLPExporter", () => { }); }); - it("should set _llmUsage side-channel for dual-write", () => { + it("should set _llmMetrics side-channel for dual-write", () => { const events = [makeGenAiEvent()]; // @ts-expect-error const $events = enrichCreatableEvents(events); const event = $events[0]; - expect(event._llmUsage).toBeDefined(); - expect(event._llmUsage.genAiSystem).toBe("openai"); - expect(event._llmUsage.responseModel).toBe("gpt-4o-2024-08-06"); - expect(event._llmUsage.inputTokens).toBe(702); - expect(event._llmUsage.outputTokens).toBe(22); - expect(event._llmUsage.totalCost).toBeCloseTo(0.001975); - expect(event._llmUsage.operationName).toBe("ai.streamText.doStream"); + expect(event._llmMetrics).toBeDefined(); + expect(event._llmMetrics.genAiSystem).toBe("openai"); + expect(event._llmMetrics.responseModel).toBe("gpt-4o-2024-08-06"); + expect(event._llmMetrics.inputTokens).toBe(702); + expect(event._llmMetrics.outputTokens).toBe(22); + expect(event._llmMetrics.totalCost).toBeCloseTo(0.001975); + expect(event._llmMetrics.operationId).toBe("ai.streamText.doStream"); }); it("should skip partial spans", () => { @@ -528,7 +528,7 @@ describe("OTLPExporter", () => { // @ts-expect-error const $events = enrichCreatableEvents(events); expect($events[0].properties["trigger.llm.total_cost"]).toBeUndefined(); - expect($events[0]._llmUsage).toBeUndefined(); + expect($events[0]._llmMetrics).toBeUndefined(); }); it("should skip spans without gen_ai.response.model or gen_ai.request.model", () => { @@ -635,8 +635,8 @@ describe("OTLPExporter", () => { // @ts-expect-error const $events = enrichCreatableEvents(events); - expect($events[0]._llmUsage.inputTokens).toBe(500); - expect($events[0]._llmUsage.outputTokens).toBe(100); + expect($events[0]._llmMetrics.inputTokens).toBe(500); + expect($events[0]._llmMetrics.outputTokens).toBe(100); }); it("should prefer gen_ai.usage.total_tokens over input+output sum", () => { @@ -659,9 +659,9 @@ describe("OTLPExporter", () => { }); // LLM usage should also use the explicit total - expect(event._llmUsage.totalTokens).toBe(200); - expect(event._llmUsage.inputTokens).toBe(100); - expect(event._llmUsage.outputTokens).toBe(50); + expect(event._llmMetrics.totalTokens).toBe(200); + expect(event._llmMetrics.inputTokens).toBe(100); + expect(event._llmMetrics.outputTokens).toBe(50); }); it("should fall back to input+output when total_tokens is absent", () => { @@ -680,7 +680,7 @@ describe("OTLPExporter", () => { text: "375", icon: "tabler-hash", }); - expect(event._llmUsage.totalTokens).toBe(375); + expect(event._llmMetrics.totalTokens).toBe(375); }); it("should use total_tokens when only total is present without input/output breakdown", () => { diff --git a/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql index 5d7f879fdd3..7cfbc0cea98 100644 --- a/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql +++ b/internal-packages/clickhouse/schema/024_create_llm_usage_v1.sql @@ -1,6 +1,7 @@ -- +goose Up -CREATE TABLE IF NOT EXISTS trigger_dev.llm_usage_v1 +CREATE TABLE IF NOT EXISTS trigger_dev.llm_metrics_v1 ( + -- Tenant context organization_id LowCardinality(String), project_id LowCardinality(String), environment_id String CODEC(ZSTD(1)), @@ -9,30 +10,45 @@ CREATE TABLE IF NOT EXISTS trigger_dev.llm_usage_v1 trace_id String CODEC(ZSTD(1)), span_id String CODEC(ZSTD(1)), + -- Model & provider gen_ai_system LowCardinality(String), request_model String CODEC(ZSTD(1)), response_model String CODEC(ZSTD(1)), matched_model_id String CODEC(ZSTD(1)), - operation_name LowCardinality(String), + operation_id LowCardinality(String), + finish_reason LowCardinality(String), + cost_source LowCardinality(String), + + -- Pricing pricing_tier_id String CODEC(ZSTD(1)), pricing_tier_name LowCardinality(String), + -- Token usage input_tokens UInt64 DEFAULT 0, output_tokens UInt64 DEFAULT 0, total_tokens UInt64 DEFAULT 0, usage_details Map(LowCardinality(String), UInt64), + -- Cost input_cost Decimal64(12) DEFAULT 0, output_cost Decimal64(12) DEFAULT 0, total_cost Decimal64(12) DEFAULT 0, cost_details Map(LowCardinality(String), Decimal64(12)), + provider_cost Decimal64(12) DEFAULT 0, + + -- Performance + ms_to_first_chunk Float64 DEFAULT 0, + tokens_per_second Float64 DEFAULT 0, + -- Attribution metadata Map(LowCardinality(String), String), + -- Timing start_time DateTime64(9) CODEC(Delta(8), ZSTD(1)), duration UInt64 DEFAULT 0 CODEC(ZSTD(1)), inserted_at DateTime64(3) DEFAULT now64(3), + -- Indexes INDEX idx_run_id run_id TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_span_id span_id TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_response_model response_model TYPE bloom_filter(0.01) GRANULARITY 1, @@ -45,4 +61,4 @@ TTL toDateTime(inserted_at) + INTERVAL 365 DAY SETTINGS ttl_only_drop_parts = 1; -- +goose Down -DROP TABLE IF EXISTS trigger_dev.llm_usage_v1; +DROP TABLE IF EXISTS trigger_dev.llm_metrics_v1; diff --git a/internal-packages/clickhouse/src/index.ts b/internal-packages/clickhouse/src/index.ts index 336abf3761f..18e52483627 100644 --- a/internal-packages/clickhouse/src/index.ts +++ b/internal-packages/clickhouse/src/index.ts @@ -27,7 +27,7 @@ import { getLogsSearchListQueryBuilder, } from "./taskEvents.js"; import { insertMetrics } from "./metrics.js"; -import { insertLlmUsage } from "./llmUsage.js"; +import { insertLlmMetrics } from "./llmMetrics.js"; import { getErrorGroups, getErrorInstances, @@ -45,7 +45,7 @@ import type { Agent as HttpsAgent } from "https"; export type * from "./taskRuns.js"; export type * from "./taskEvents.js"; export type * from "./metrics.js"; -export type * from "./llmUsage.js"; +export type * from "./llmMetrics.js"; export type * from "./errors.js"; export type * from "./client/queryBuilder.js"; @@ -227,9 +227,9 @@ export class ClickHouse { }; } - get llmUsage() { + get llmMetrics() { return { - insert: insertLlmUsage(this.writer), + insert: insertLlmMetrics(this.writer), }; } diff --git a/internal-packages/clickhouse/src/llmUsage.ts b/internal-packages/clickhouse/src/llmMetrics.ts similarity index 64% rename from internal-packages/clickhouse/src/llmUsage.ts rename to internal-packages/clickhouse/src/llmMetrics.ts index e9423962ac4..1f830b707d8 100644 --- a/internal-packages/clickhouse/src/llmUsage.ts +++ b/internal-packages/clickhouse/src/llmMetrics.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import { ClickhouseWriter } from "./client/types.js"; -export const LlmUsageV1Input = z.object({ +export const LlmMetricsV1Input = z.object({ organization_id: z.string(), project_id: z.string(), environment_id: z.string(), @@ -14,7 +14,10 @@ export const LlmUsageV1Input = z.object({ request_model: z.string(), response_model: z.string(), matched_model_id: z.string(), - operation_name: z.string(), + operation_id: z.string(), + finish_reason: z.string(), + cost_source: z.string(), + pricing_tier_id: z.string(), pricing_tier_name: z.string(), @@ -27,6 +30,10 @@ export const LlmUsageV1Input = z.object({ output_cost: z.number(), total_cost: z.number(), cost_details: z.record(z.string(), z.number()), + provider_cost: z.number(), + + ms_to_first_chunk: z.number(), + tokens_per_second: z.number(), metadata: z.record(z.string(), z.string()), @@ -34,11 +41,11 @@ export const LlmUsageV1Input = z.object({ duration: z.string(), }); -export type LlmUsageV1Input = z.input; +export type LlmMetricsV1Input = z.input; -export function insertLlmUsage(ch: ClickhouseWriter) { - return ch.insertUnsafe({ - name: "insertLlmUsage", - table: "trigger_dev.llm_usage_v1", +export function insertLlmMetrics(ch: ClickhouseWriter) { + return ch.insertUnsafe({ + name: "insertLlmMetrics", + table: "trigger_dev.llm_metrics_v1", }); } From 5c7daeb6b4431e0f983012a5bbc8f90c608c01f3 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Sun, 15 Mar 2026 22:14:56 +0000 Subject: [PATCH 16/30] Create AI/LLM metrics dashboard, with support for a model filter --- .../app/components/code/QueryResultsChart.tsx | 5 + .../app/components/metrics/ModelsFilter.tsx | 159 ++++++++++ .../presenters/v3/BuiltInDashboards.server.ts | 293 +++++++++++++++++- .../v3/MetricDashboardPresenter.server.ts | 4 + .../route.tsx | 53 +++- apps/webapp/app/routes/resources.metric.tsx | 4 + .../app/services/queryService.server.ts | 7 + apps/webapp/app/utils/columnFormat.ts | 17 +- apps/webapp/app/v3/querySchemas.ts | 44 +++ apps/webapp/seed-ai-spans.mts | 26 +- 10 files changed, 601 insertions(+), 11 deletions(-) create mode 100644 apps/webapp/app/components/metrics/ModelsFilter.tsx diff --git a/apps/webapp/app/components/code/QueryResultsChart.tsx b/apps/webapp/app/components/code/QueryResultsChart.tsx index aa6bc6d8898..2c3f1c9f2bf 100644 --- a/apps/webapp/app/components/code/QueryResultsChart.tsx +++ b/apps/webapp/app/components/code/QueryResultsChart.tsx @@ -1217,6 +1217,11 @@ function createYAxisFormatter( if (format === "costInDollars" || format === "cost") { return (value: number): string => { const dollars = format === "cost" ? value / 100 : value; + if (dollars === 0) return "$0"; + if (Math.abs(dollars) >= 1000) return `$${(dollars / 1000).toFixed(1)}K`; + if (Math.abs(dollars) >= 1) return `$${dollars.toFixed(2)}`; + if (Math.abs(dollars) >= 0.01) return `$${dollars.toFixed(4)}`; + if (Math.abs(dollars) >= 0.0001) return `$${dollars.toFixed(6)}`; return formatCurrencyAccurate(dollars); }; } diff --git a/apps/webapp/app/components/metrics/ModelsFilter.tsx b/apps/webapp/app/components/metrics/ModelsFilter.tsx new file mode 100644 index 00000000000..6250cd20c99 --- /dev/null +++ b/apps/webapp/app/components/metrics/ModelsFilter.tsx @@ -0,0 +1,159 @@ +import { CubeIcon } from "@heroicons/react/20/solid"; +import * as Ariakit from "@ariakit/react"; +import { type ReactNode, useMemo } from "react"; +import { AppliedFilter } from "~/components/primitives/AppliedFilter"; +import { + ComboBox, + SelectItem, + SelectList, + SelectPopover, + SelectProvider, + SelectTrigger, +} from "~/components/primitives/Select"; +import { useSearchParams } from "~/hooks/useSearchParam"; +import { appliedSummary, FilterMenuProvider } from "~/components/runs/v3/SharedFilters"; +import { tablerIcons } from "~/utils/tablerIcons"; +import tablerSpritePath from "~/components/primitives/tabler-sprite.svg"; +import { AnthropicLogoIcon } from "~/assets/icons/AnthropicLogoIcon"; + +const shortcut = { key: "m" }; + +export type ModelOption = { + model: string; + system: string; +}; + +interface ModelsFilterProps { + possibleModels: ModelOption[]; +} + +function modelIcon(system: string, model: string): ReactNode { + // For gateway/openrouter, derive provider from model prefix + let provider = system.split(".")[0]; + if (provider === "gateway" || provider === "openrouter") { + if (model.includes("/")) { + provider = model.split("/")[0].replace(/-/g, ""); + } + } + + // Special case: Anthropic uses a custom SVG icon + if (provider === "anthropic") { + return ; + } + + const iconName = `tabler-brand-${provider}`; + if (tablerIcons.has(iconName)) { + return ( + + + + ); + } + + return ; +} + +export function ModelsFilter({ possibleModels }: ModelsFilterProps) { + const { values, replace, del } = useSearchParams(); + const selectedModels = values("models"); + + if (selectedModels.length === 0 || selectedModels.every((v) => v === "")) { + return ( + + {(search, setSearch) => ( + } + variant="secondary/small" + shortcut={shortcut} + tooltipTitle="Filter by model" + > + Models + + } + searchValue={search} + clearSearchValue={() => setSearch("")} + possibleModels={possibleModels} + /> + )} + + ); + } + + return ( + + {(search, setSearch) => ( + }> + } + value={appliedSummary(selectedModels)} + onRemove={() => del(["models"])} + variant="secondary/small" + /> + + } + searchValue={search} + clearSearchValue={() => setSearch("")} + possibleModels={possibleModels} + /> + )} + + ); +} + +function ModelsDropdown({ + trigger, + clearSearchValue, + searchValue, + onClose, + possibleModels, +}: { + trigger: ReactNode; + clearSearchValue: () => void; + searchValue: string; + onClose?: () => void; + possibleModels: ModelOption[]; +}) { + const { values, replace } = useSearchParams(); + + const handleChange = (values: string[]) => { + clearSearchValue(); + replace({ models: values }); + }; + + const filtered = useMemo(() => { + return possibleModels.filter((m) => { + return m.model?.toLowerCase().includes(searchValue.toLowerCase()); + }); + }, [searchValue, possibleModels]); + + return ( + + {trigger} + { + if (onClose) { + onClose(); + return false; + } + return true; + }} + > + + + {filtered.map((m) => ( + + {m.model} + + ))} + {filtered.length === 0 && No models found} + + + + ); +} diff --git a/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts b/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts index da95eeacc00..144176c88fb 100644 --- a/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts +++ b/apps/webapp/app/presenters/v3/BuiltInDashboards.server.ts @@ -4,6 +4,7 @@ import { z } from "zod"; const overviewDashboard: BuiltInDashboard = { key: "overview", title: "Metrics", + filters: ["tasks", "queues"], layout: { version: "1", layout: [ @@ -213,7 +214,297 @@ const overviewDashboard: BuiltInDashboard = { }, }; -const builtInDashboards: BuiltInDashboard[] = [overviewDashboard]; +const llmDashboard: BuiltInDashboard = { + key: "llm", + title: "AI Metrics", + filters: ["tasks", "models"], + layout: { + version: "1", + layout: [ + // Big numbers row + { i: "llm-cost", x: 0, y: 0, w: 3, h: 4 }, + { i: "llm-calls", x: 3, y: 0, w: 3, h: 4 }, + { i: "llm-ttfc", x: 6, y: 0, w: 3, h: 4 }, + { i: "llm-tps", x: 9, y: 0, w: 3, h: 4 }, + // Cost section + { i: "llm-title-cost", x: 0, y: 4, w: 12, h: 2, minH: 2, maxH: 2 }, + { i: "llm-cost-time", x: 0, y: 6, w: 6, h: 13 }, + { i: "llm-cost-model", x: 6, y: 6, w: 6, h: 13 }, + // Usage section + { i: "llm-title-usage", x: 0, y: 19, w: 12, h: 2, minH: 2, maxH: 2 }, + { i: "llm-tokens-time", x: 0, y: 21, w: 6, h: 13 }, + { i: "llm-calls-model", x: 6, y: 21, w: 6, h: 13 }, + // Performance section + { i: "llm-title-perf", x: 0, y: 34, w: 12, h: 2, minH: 2, maxH: 2 }, + { i: "llm-ttfc-time", x: 0, y: 36, w: 6, h: 13 }, + { i: "llm-tps-model", x: 6, y: 36, w: 6, h: 13 }, + { i: "llm-latency-pct", x: 0, y: 49, w: 6, h: 13 }, + { i: "llm-latency-time", x: 6, y: 49, w: 6, h: 13 }, + // Behavior section + { i: "llm-title-behavior", x: 0, y: 62, w: 12, h: 2, minH: 2, maxH: 2 }, + { i: "llm-finish-reasons", x: 0, y: 64, w: 6, h: 13 }, + { i: "llm-top-runs", x: 6, y: 64, w: 6, h: 13 }, + // Attribution section + { i: "llm-title-attribution", x: 0, y: 77, w: 12, h: 2, minH: 2, maxH: 2 }, + { i: "llm-cost-task", x: 0, y: 79, w: 6, h: 13 }, + { i: "llm-cost-provider", x: 6, y: 79, w: 6, h: 13 }, + { i: "llm-cost-user", x: 0, y: 92, w: 12, h: 13 }, + // Efficiency section + { i: "llm-title-efficiency", x: 0, y: 105, w: 12, h: 2, minH: 2, maxH: 2 }, + { i: "llm-cost-operation", x: 0, y: 107, w: 6, h: 13 }, + { i: "llm-cache-util", x: 6, y: 107, w: 6, h: 13 }, + ], + widgets: { + "llm-cost": { + title: "Total LLM cost", + query: "SELECT\r\n SUM(total_cost) AS total_cost\r\nFROM\r\n llm_metrics", + display: { + type: "bignumber", + column: "total_cost", + aggregation: "sum", + abbreviate: true, + }, + }, + "llm-calls": { + title: "Total calls", + query: "SELECT\r\n count() AS total_calls\r\nFROM\r\n llm_metrics", + display: { + type: "bignumber", + column: "total_calls", + aggregation: "sum", + abbreviate: false, + }, + }, + "llm-ttfc": { + title: "Avg TTFC", + query: + "SELECT\r\n round(avg(ms_to_first_chunk), 1) AS avg_ttfc\r\nFROM\r\n llm_metrics\r\nWHERE ms_to_first_chunk > 0", + display: { + type: "bignumber", + column: "avg_ttfc", + aggregation: "avg", + abbreviate: false, + suffix: "ms", + }, + }, + "llm-tps": { + title: "Avg tokens/sec", + query: + "SELECT\r\n round(avg(tokens_per_second), 1) AS avg_tps\r\nFROM\r\n llm_metrics\r\nWHERE tokens_per_second > 0", + display: { + type: "bignumber", + column: "avg_tps", + aggregation: "avg", + abbreviate: false, + suffix: "/s", + }, + }, + "llm-title-cost": { title: "Cost", query: "", display: { type: "title" } }, + "llm-cost-time": { + title: "Cost over time", + query: + "SELECT\r\n timeBucket(),\r\n SUM(total_cost) AS total_cost\r\nFROM\r\n llm_metrics\r\nGROUP BY\r\n timeBucket\r\nORDER BY\r\n timeBucket", + display: { + type: "chart", + chartType: "line", + xAxisColumn: "timebucket", + yAxisColumns: ["total_cost"], + groupByColumn: null, + stacked: false, + sortByColumn: null, + sortDirection: "asc", + aggregation: "sum", + }, + }, + "llm-cost-model": { + title: "Cost by model", + query: + "SELECT\r\n timeBucket(),\r\n response_model,\r\n SUM(total_cost) AS total_cost\r\nFROM\r\n llm_metrics\r\nGROUP BY\r\n timeBucket,\r\n response_model\r\nORDER BY\r\n timeBucket", + display: { + type: "chart", + chartType: "bar", + xAxisColumn: "timebucket", + yAxisColumns: ["total_cost"], + groupByColumn: "response_model", + stacked: true, + sortByColumn: null, + sortDirection: "asc", + aggregation: "sum", + }, + }, + "llm-title-usage": { title: "Usage", query: "", display: { type: "title" } }, + "llm-tokens-time": { + title: "Tokens over time", + query: + "SELECT\r\n timeBucket(),\r\n SUM(input_tokens) AS input_tokens,\r\n SUM(output_tokens) AS output_tokens\r\nFROM\r\n llm_metrics\r\nGROUP BY\r\n timeBucket\r\nORDER BY\r\n timeBucket", + display: { + type: "chart", + chartType: "bar", + xAxisColumn: "timebucket", + yAxisColumns: ["input_tokens", "output_tokens"], + groupByColumn: null, + stacked: true, + sortByColumn: null, + sortDirection: "asc", + aggregation: "sum", + }, + }, + "llm-calls-model": { + title: "Calls by model", + query: + "SELECT\r\n response_model,\r\n count() AS calls,\r\n SUM(total_tokens) AS tokens,\r\n SUM(total_cost) AS cost\r\nFROM\r\n llm_metrics\r\nGROUP BY\r\n response_model\r\nORDER BY\r\n cost DESC", + display: { type: "table", prettyFormatting: true, sorting: [] }, + }, + "llm-title-perf": { title: "Performance", query: "", display: { type: "title" } }, + "llm-ttfc-time": { + title: "TTFC over time", + query: + "SELECT\r\n timeBucket(),\r\n round(avg(ms_to_first_chunk), 1) AS avg_ttfc\r\nFROM\r\n llm_metrics\r\nWHERE ms_to_first_chunk > 0\r\nGROUP BY\r\n timeBucket\r\nORDER BY\r\n timeBucket", + display: { + type: "chart", + chartType: "line", + xAxisColumn: "timebucket", + yAxisColumns: ["avg_ttfc"], + groupByColumn: null, + stacked: false, + sortByColumn: null, + sortDirection: "asc", + aggregation: "avg", + }, + }, + "llm-tps-model": { + title: "Tokens/sec by model", + query: + "SELECT\r\n timeBucket(),\r\n response_model,\r\n round(avg(tokens_per_second), 1) AS avg_tps\r\nFROM\r\n llm_metrics\r\nWHERE tokens_per_second > 0\r\nGROUP BY\r\n timeBucket,\r\n response_model\r\nORDER BY\r\n timeBucket", + display: { + type: "chart", + chartType: "line", + xAxisColumn: "timebucket", + yAxisColumns: ["avg_tps"], + groupByColumn: "response_model", + stacked: false, + sortByColumn: null, + sortDirection: "asc", + aggregation: "avg", + }, + }, + "llm-latency-pct": { + title: "Latency percentiles by model", + query: + "SELECT\r\n response_model,\r\n round(quantile(0.5)(ms_to_first_chunk), 1) AS p50,\r\n round(quantile(0.9)(ms_to_first_chunk), 1) AS p90,\r\n round(quantile(0.95)(ms_to_first_chunk), 1) AS p95,\r\n round(quantile(0.99)(ms_to_first_chunk), 1) AS p99,\r\n count() AS calls\r\nFROM\r\n llm_metrics\r\nWHERE ms_to_first_chunk > 0\r\nGROUP BY\r\n response_model\r\nORDER BY\r\n p50 DESC", + display: { type: "table", prettyFormatting: true, sorting: [] }, + }, + "llm-latency-time": { + title: "Latency percentiles over time", + query: + "SELECT\r\n timeBucket(),\r\n round(quantile(0.5)(ms_to_first_chunk), 1) AS p50,\r\n round(quantile(0.95)(ms_to_first_chunk), 1) AS p95\r\nFROM\r\n llm_metrics\r\nWHERE ms_to_first_chunk > 0\r\nGROUP BY\r\n timeBucket\r\nORDER BY\r\n timeBucket", + display: { + type: "chart", + chartType: "line", + xAxisColumn: "timebucket", + yAxisColumns: ["p50", "p95"], + groupByColumn: null, + stacked: false, + sortByColumn: null, + sortDirection: "asc", + aggregation: "avg", + seriesColors: { p95: "#f43f5e" }, + }, + }, + "llm-title-behavior": { title: "Behavior", query: "", display: { type: "title" } }, + "llm-finish-reasons": { + title: "Finish reasons over time", + query: + "SELECT\r\n timeBucket(),\r\n finish_reason,\r\n count() AS calls\r\nFROM\r\n llm_metrics\r\nWHERE finish_reason != ''\r\nGROUP BY\r\n timeBucket,\r\n finish_reason\r\nORDER BY\r\n timeBucket", + display: { + type: "chart", + chartType: "bar", + xAxisColumn: "timebucket", + yAxisColumns: ["calls"], + groupByColumn: "finish_reason", + stacked: true, + sortByColumn: null, + sortDirection: "asc", + aggregation: "sum", + }, + }, + "llm-top-runs": { + title: "Most expensive runs", + query: + "SELECT\r\n run_id,\r\n task_identifier,\r\n SUM(total_cost) AS llm_cost,\r\n SUM(total_tokens) AS tokens\r\nFROM\r\n llm_metrics\r\nGROUP BY\r\n run_id,\r\n task_identifier\r\nORDER BY\r\n llm_cost DESC\r\nLIMIT 50", + display: { type: "table", prettyFormatting: true, sorting: [] }, + }, + "llm-title-attribution": { title: "Attribution", query: "", display: { type: "title" } }, + "llm-cost-task": { + title: "Cost by task", + query: + "SELECT\r\n task_identifier,\r\n SUM(total_cost) AS cost,\r\n SUM(total_tokens) AS tokens,\r\n count() AS calls\r\nFROM\r\n llm_metrics\r\nGROUP BY\r\n task_identifier\r\nORDER BY\r\n cost DESC", + display: { type: "table", prettyFormatting: true, sorting: [] }, + }, + "llm-cost-provider": { + title: "Cost by provider", + query: + "SELECT\r\n timeBucket(),\r\n gen_ai_system,\r\n SUM(total_cost) AS total_cost\r\nFROM\r\n llm_metrics\r\nGROUP BY\r\n timeBucket,\r\n gen_ai_system\r\nORDER BY\r\n timeBucket", + display: { + type: "chart", + chartType: "bar", + xAxisColumn: "timebucket", + yAxisColumns: ["total_cost"], + groupByColumn: "gen_ai_system", + stacked: true, + sortByColumn: null, + sortDirection: "asc", + aggregation: "sum", + }, + }, + "llm-cost-user": { + title: "Cost by user", + query: + "SELECT\r\n metadata['userId'] AS user_id,\r\n SUM(total_cost) AS cost,\r\n SUM(total_tokens) AS tokens,\r\n count() AS calls\r\nFROM\r\n llm_metrics\r\nWHERE metadata['userId'] != ''\r\nGROUP BY\r\n user_id\r\nORDER BY\r\n cost DESC\r\nLIMIT 20", + display: { type: "table", prettyFormatting: true, sorting: [] }, + }, + "llm-title-efficiency": { title: "Efficiency", query: "", display: { type: "title" } }, + "llm-cost-operation": { + title: "Cost by operation type", + query: + "SELECT\r\n timeBucket(),\r\n operation_id,\r\n SUM(total_cost) AS total_cost\r\nFROM\r\n llm_metrics\r\nWHERE operation_id != ''\r\nGROUP BY\r\n timeBucket,\r\n operation_id\r\nORDER BY\r\n timeBucket", + display: { + type: "chart", + chartType: "bar", + xAxisColumn: "timebucket", + yAxisColumns: ["total_cost"], + groupByColumn: "operation_id", + stacked: true, + sortByColumn: null, + sortDirection: "asc", + aggregation: "sum", + }, + }, + "llm-cache-util": { + title: "Cache utilization", + query: + "SELECT\r\n timeBucket(),\r\n round(countIf(cached_read_tokens > 0) * 100.0 / count(), 1) AS cache_hit_pct,\r\n round(avg(cached_read_tokens), 0) AS avg_cached_tokens\r\nFROM\r\n llm_metrics\r\nGROUP BY\r\n timeBucket\r\nORDER BY\r\n timeBucket", + display: { + type: "chart", + chartType: "line", + xAxisColumn: "timebucket", + yAxisColumns: ["cache_hit_pct"], + groupByColumn: null, + stacked: false, + sortByColumn: null, + sortDirection: "asc", + aggregation: "avg", + }, + }, + }, + }, +}; + +const builtInDashboards: BuiltInDashboard[] = [overviewDashboard, llmDashboard]; + +export function builtInDashboardList(): BuiltInDashboard[] { + return builtInDashboards; +} export function builtInDashboard(key: string): BuiltInDashboard { const dashboard = builtInDashboards.find((d) => d.key === key); diff --git a/apps/webapp/app/presenters/v3/MetricDashboardPresenter.server.ts b/apps/webapp/app/presenters/v3/MetricDashboardPresenter.server.ts index d5363ddb9af..90b60891c58 100644 --- a/apps/webapp/app/presenters/v3/MetricDashboardPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/MetricDashboardPresenter.server.ts @@ -58,10 +58,14 @@ export type CustomDashboard = { defaultPeriod: string; }; +export type BuiltInDashboardFilter = "tasks" | "queues" | "models"; + export type BuiltInDashboard = { key: string; title: string; layout: DashboardLayout; + /** Which filters to show in the toolbar. Defaults to ["tasks", "queues"] if not specified. */ + filters?: BuiltInDashboardFilter[]; }; /** Returns the dashboard layout */ diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx index 1606158598a..1374fd0f3de 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx @@ -6,6 +6,7 @@ import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { LogsTaskFilter } from "~/components/logs/LogsTaskFilter"; +import { ModelsFilter, type ModelOption } from "~/components/metrics/ModelsFilter"; import { type WidgetData } from "~/components/metrics/QueryWidget"; import { QueuesFilter } from "~/components/metrics/QueuesFilter"; import { ScopeFilter } from "~/components/metrics/ScopeFilter"; @@ -22,10 +23,12 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { getAllTaskIdentifiers } from "~/models/task.server"; import { + type BuiltInDashboardFilter, type LayoutItem, type Widget, MetricDashboardPresenter, } from "~/presenters/v3/MetricDashboardPresenter.server"; +import { clickhouseClient } from "~/services/clickhouseInstance.server"; import { requireUser } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { EnvironmentParamSchema } from "~/utils/pathBuilder"; @@ -66,11 +69,38 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { getAllTaskIdentifiers($replica, environment.id), ]); + const filters = dashboard.filters ?? ["tasks", "queues"]; + + // Load distinct models from ClickHouse if the dashboard has a models filter + let possibleModels: { model: string; system: string }[] = []; + if (filters.includes("models")) { + const queryFn = clickhouseClient.reader.query({ + name: "getDistinctModels", + query: `SELECT response_model, any(gen_ai_system) AS gen_ai_system FROM trigger_dev.llm_metrics_v1 WHERE organization_id = {organizationId: String} AND project_id = {projectId: String} AND environment_id = {environmentId: String} AND response_model != '' GROUP BY response_model ORDER BY response_model`, + params: z.object({ + organizationId: z.string(), + projectId: z.string(), + environmentId: z.string(), + }), + schema: z.object({ response_model: z.string(), gen_ai_system: z.string() }), + }); + const [error, rows] = await queryFn({ + organizationId: project.organizationId, + projectId: project.id, + environmentId: environment.id, + }); + if (!error) { + possibleModels = rows.map((r) => ({ model: r.response_model, system: r.gen_ai_system })); + } + } + return typedjson({ ...dashboard, + filters, possibleTasks: possibleTasks .map((task) => ({ slug: task.slug, triggerSource: task.triggerSource })) .sort((a, b) => a.slug.localeCompare(b.slug)), + possibleModels, }); }; @@ -80,7 +110,9 @@ export default function Page() { title, layout: dashboardLayout, defaultPeriod, + filters, possibleTasks, + possibleModels, } = useTypedLoaderData(); const organization = useOrganization(); @@ -107,7 +139,9 @@ export default function Page() { widgets={dashboardLayout.widgets} defaultPeriod={defaultPeriod} editable={false} + filters={filters} possibleTasks={possibleTasks} + possibleModels={possibleModels} />
@@ -120,7 +154,9 @@ export function MetricDashboard({ widgets, defaultPeriod, editable, + filters: filterConfig, possibleTasks, + possibleModels, onLayoutChange, onEditWidget, onRenameWidget, @@ -133,8 +169,12 @@ export function MetricDashboard({ widgets: Record; defaultPeriod: string; editable: boolean; + /** Which filters to show. Defaults to ["tasks", "queues"]. */ + filters?: BuiltInDashboardFilter[]; /** Possible tasks for filtering */ possibleTasks?: { slug: string; triggerSource: TaskTriggerSource }[]; + /** Possible models for filtering */ + possibleModels?: ModelOption[]; onLayoutChange?: (layout: LayoutItem[]) => void; onEditWidget?: (widgetId: string, widget: WidgetData) => void; onRenameWidget?: (widgetId: string, newTitle: string) => void; @@ -161,6 +201,9 @@ export function MetricDashboard({ const scope = parsedScope.success ? parsedScope.data : "environment"; const tasks = values("tasks").filter((v) => v !== ""); const queues = values("queues").filter((v) => v !== ""); + const models = values("models").filter((v) => v !== ""); + + const activeFilters = filterConfig ?? ["tasks", "queues"]; const handleLayoutChange = useCallback( (newLayout: readonly LayoutItem[]) => { @@ -187,8 +230,13 @@ export function MetricDashboard({
- - + {activeFilters.includes("tasks") && ( + + )} + {activeFilters.includes("queues") && } + {activeFilters.includes("models") && ( + + )} 0 ? tasks : undefined} queues={queues.length > 0 ? queues : undefined} + responseModels={models.length > 0 ? models : undefined} config={widget.display} organizationId={organization.id} projectId={project.id} diff --git a/apps/webapp/app/routes/resources.metric.tsx b/apps/webapp/app/routes/resources.metric.tsx index 3c19d3947f9..60d83f7a600 100644 --- a/apps/webapp/app/routes/resources.metric.tsx +++ b/apps/webapp/app/routes/resources.metric.tsx @@ -44,6 +44,7 @@ const MetricWidgetQuery = z.object({ to: z.string().nullable(), taskIdentifiers: z.array(z.string()).optional(), queues: z.array(z.string()).optional(), + responseModels: z.array(z.string()).optional(), tags: z.array(z.string()).optional(), }); @@ -74,6 +75,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { to, taskIdentifiers, queues, + responseModels, tags, } = submission.data; @@ -107,6 +109,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { to, taskIdentifiers, queues, + responseModels, // Set higher concurrency if many widgets are on screen at once customOrgConcurrencyLimit: env.METRIC_WIDGET_DEFAULT_ORG_CONCURRENCY_LIMIT, }); @@ -257,6 +260,7 @@ export function MetricWidget({ props.scope, JSON.stringify(props.taskIdentifiers), JSON.stringify(props.queues), + JSON.stringify(props.responseModels), ]); const data = response?.success diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts index 6cd2af03b16..ce3444902d8 100644 --- a/apps/webapp/app/services/queryService.server.ts +++ b/apps/webapp/app/services/queryService.server.ts @@ -73,6 +73,8 @@ export type ExecuteQueryOptions = Omit< taskIdentifiers?: string[]; /** Filter to specific queues */ queues?: string[]; + /** Filter to specific response models */ + responseModels?: string[]; /** History options for saving query to billing/audit */ history?: { /** Where the query originated from */ @@ -127,6 +129,7 @@ export async function executeQuery( environmentId, taskIdentifiers, queues, + responseModels, history, customOrgConcurrencyLimit, ...baseOptions @@ -210,6 +213,10 @@ export async function executeQuery( ? { op: "in", values: taskIdentifiers } : undefined, queue: queues && queues.length > 0 ? { op: "in", values: queues } : undefined, + response_model: + responseModels && responseModels.length > 0 + ? { op: "in", values: responseModels } + : undefined, } satisfies Record; // Compute the effective time range for timeBucket() interval calculation diff --git a/apps/webapp/app/utils/columnFormat.ts b/apps/webapp/app/utils/columnFormat.ts index 731b42fdb9f..c11226123ab 100644 --- a/apps/webapp/app/utils/columnFormat.ts +++ b/apps/webapp/app/utils/columnFormat.ts @@ -39,6 +39,19 @@ export function formatQuantity(value: number): string { return value.toLocaleString(); } +/** + * Format a dollar amount with adaptive precision — avoids trailing zeros. + */ +function formatCostAdaptive(dollars: number): string { + if (dollars === 0) return "$0"; + const abs = Math.abs(dollars); + if (abs >= 1000) return `$${dollars.toFixed(2)}`; + if (abs >= 1) return `$${dollars.toFixed(2)}`; + if (abs >= 0.01) return `$${dollars.toFixed(4)}`; + if (abs >= 0.0001) return `$${dollars.toFixed(6)}`; + return formatCurrencyAccurate(dollars); +} + /** * Creates a value formatter function for a given column format type. * Used by chart tooltips, legend values, and big number cards. @@ -61,9 +74,9 @@ export function createValueFormatter( case "durationSeconds": return (v) => formatDurationMilliseconds(v * 1000, { style: "short" }); case "costInDollars": - return (v) => formatCurrencyAccurate(v); + return (v) => formatCostAdaptive(v); case "cost": - return (v) => formatCurrencyAccurate(v / 100); + return (v) => formatCostAdaptive(v / 100); default: return undefined; } diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts index b4f4fceb80c..6a6c8758fdb 100644 --- a/apps/webapp/app/v3/querySchemas.ts +++ b/apps/webapp/app/v3/querySchemas.ts @@ -712,6 +712,33 @@ export const llmMetricsSchema: TableSchema = { example: "724", }), }, + cached_read_tokens: { + name: "cached_read_tokens", + ...column("UInt64", { + description: + "Input tokens served from the provider's prompt cache (cheaper than regular input tokens). Supported by Anthropic and OpenAI.", + example: "8200", + }), + expression: "usage_details['input_cached_tokens']", + }, + cache_creation_tokens: { + name: "cache_creation_tokens", + ...column("UInt64", { + description: + "Input tokens written to create a new prompt cache entry. Supported by Anthropic.", + example: "1751", + }), + expression: "usage_details['cache_creation_input_tokens']", + }, + reasoning_tokens: { + name: "reasoning_tokens", + ...column("UInt64", { + description: + "Tokens used for chain-of-thought reasoning (e.g. OpenAI o-series, DeepSeek R1). These count toward output but are not visible in the response.", + example: "512", + }), + expression: "usage_details['reasoning_tokens']", + }, input_cost: { name: "input_cost", ...column("Decimal64(12)", { @@ -734,6 +761,23 @@ export const llmMetricsSchema: TableSchema = { coreColumn: true, }), }, + cached_read_cost: { + name: "cached_read_cost", + ...column("Decimal64(12)", { + description: + "Cost of cached input tokens (discounted vs regular input). Only present when the pricing tier has a separate cached input price.", + customRenderType: "costInDollars", + }), + expression: "cost_details['input_cached_tokens']", + }, + cache_creation_cost: { + name: "cache_creation_cost", + ...column("Decimal64(12)", { + description: "Cost of tokens written to create a prompt cache entry.", + customRenderType: "costInDollars", + }), + expression: "cost_details['cache_creation_input_tokens']", + }, provider_cost: { name: "provider_cost", ...column("Decimal64(12)", { diff --git a/apps/webapp/seed-ai-spans.mts b/apps/webapp/seed-ai-spans.mts index ef6a8f4b6ec..8799e6799b6 100644 --- a/apps/webapp/seed-ai-spans.mts +++ b/apps/webapp/seed-ai-spans.mts @@ -25,6 +25,16 @@ const TASK_SLUG = "ai-chat"; const QUEUE_NAME = "task/ai-chat"; const WORKER_VERSION = "seed-ai-spans-v1"; +const SEED_USER_IDS = [ + "user_alice", "user_bob", "user_carol", "user_dave", + "user_eve", "user_frank", "user_grace", "user_heidi", + "user_ivan", "user_judy", "user_karl", "user_liam", +]; + +function randomUserId(): string { + return SEED_USER_IDS[Math.floor(Math.random() * SEED_USER_IDS.length)]; +} + // --------------------------------------------------------------------------- // ClickHouse formatting helpers (replicated from clickhouseEventRepository) // --------------------------------------------------------------------------- @@ -158,8 +168,9 @@ function eventToLlmMetricsRow(event: CreateEventInput): LlmMetricsV1Input { // --------------------------------------------------------------------------- async function seedAiSpans() { + const seedUserId = randomUserId(); crumb("seed started"); // @crumbs - console.log("Starting AI span seed...\n"); + console.log(`Starting AI span seed (userId: ${seedUserId})...\n`); // 1. Find user crumb("finding user"); // @crumbs @@ -301,7 +312,7 @@ async function seedAiSpans() { lockedToVersionId: worker.id, startedAt, completedAt, - runTags: ["user:seed_user_42", "chat:seed_session"], + runTags: [`user:${seedUserId}`, "chat:seed_session"], taskEventStore: "clickhouse_v2", }, }); @@ -320,6 +331,7 @@ async function seedAiSpans() { organizationId: org.id, taskSlug: TASK_SLUG, baseTimeMs: now, + seedUserId, }); crumb("span tree built", { spanCount: events.length }); // @crumbs @@ -398,6 +410,7 @@ type SpanTreeParams = { organizationId: string; taskSlug: string; baseTimeMs: number; + seedUserId: string; }; function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { @@ -410,10 +423,11 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { organizationId, taskSlug, baseTimeMs, + seedUserId, } = params; const events: CreateEventInput[] = []; - const runTags = ["user:seed_user_42", "chat:seed_session"]; + const runTags = [`user:${seedUserId}`, "chat:seed_session"]; // Timing cursor — each span advances this let cursor = baseTimeMs; @@ -591,7 +605,7 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { "ai.usage.inputTokens": 807, "ai.usage.outputTokens": 242, "ai.usage.totalTokens": 1049, - "ai.telemetry.metadata.userId": "seed_user_42", + "ai.telemetry.metadata.userId": seedUserId, "ai.telemetry.functionId": "ai-chat", "operation.name": "ai.streamText", }, @@ -630,7 +644,7 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { "ai.usage.inputTokens": 284, "ai.usage.outputTokens": 55, "ai.usage.totalTokens": 339, - "ai.telemetry.metadata.userId": "seed_user_42", + "ai.telemetry.metadata.userId": seedUserId, "ai.telemetry.functionId": "ai-chat", "operation.name": "ai.streamText.doStream", }, @@ -686,7 +700,7 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { "ai.usage.outputTokens": 187, "ai.usage.totalTokens": 710, "ai.usage.reasoningTokens": 42, - "ai.telemetry.metadata.userId": "seed_user_42", + "ai.telemetry.metadata.userId": seedUserId, "ai.telemetry.functionId": "ai-chat", "operation.name": "ai.streamText.doStream", }, From 4092b70f8fe05d9d5447c0d9cf043162f7d2f44e Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Tue, 17 Mar 2026 10:28:24 +0000 Subject: [PATCH 17/30] Lots more example content in the spans seed --- apps/webapp/seed-ai-spans.mts | 272 ++++++++++++++++++++++++++++++++-- 1 file changed, 257 insertions(+), 15 deletions(-) diff --git a/apps/webapp/seed-ai-spans.mts b/apps/webapp/seed-ai-spans.mts index 8799e6799b6..cc701843d31 100644 --- a/apps/webapp/seed-ai-spans.mts +++ b/apps/webapp/seed-ai-spans.mts @@ -300,7 +300,57 @@ async function seedAiSpans() { status: "COMPLETED_SUCCESSFULLY", taskIdentifier: TASK_SLUG, payload: JSON.stringify({ - message: "What is the current Federal Reserve interest rate?", + message: `I need a comprehensive analysis of the current Federal Reserve interest rate policy and its broader economic implications. Please cover all of the following areas in detail: + +## 1. Current Rate Policy + +- What is the **current federal funds target rate** range? +- When was it last changed, and by how much? +- What was the FOMC vote breakdown — were there any dissents? +- What key language changes appeared in the most recent FOMC statement compared to the prior meeting? + +## 2. Rate History & Trajectory + +- Provide a complete timeline of rate decisions over the past **18 months**, including the size of each move +- How does the current rate compare to the **pre-pandemic neutral rate** estimate? +- What does the latest **dot plot** (Summary of Economic Projections) show for 2025, 2026, and the longer-run rate? +- How has the **median longer-run rate estimate** shifted over the past year? + +## 3. Inflation & Economic Data Context + +- What are the latest readings for **Core PCE**, **headline CPI**, and **trimmed mean CPI**? +- How does current inflation compare to the Fed's 2% symmetric target? +- What does the **breakeven inflation rate** (5-year and 10-year TIPS spreads) suggest about market inflation expectations? +- Are there any notable divergences between goods inflation and services inflation? + +## 4. Labor Market Assessment + +- What is the current **unemployment rate**, and how has it trended over the past 6 months? +- What do **nonfarm payrolls**, **JOLTs job openings**, and **initial jobless claims** indicate about labor market health? +- Is wage growth (via the **Employment Cost Index** and **Average Hourly Earnings**) still running above levels consistent with 2% inflation? +- How does the Fed view the balance between its **maximum employment** and **price stability** mandates right now? + +## 5. Forward Guidance & Market Expectations + +- What are the upcoming **FOMC meeting dates** for the next 6 months? +- What does the **CME FedWatch Tool** show for the probability of rate changes at each upcoming meeting? +- How do **fed funds futures** and **OIS swaps** price the terminal rate for this cycle? +- Are there any notable divergences between Fed guidance and market pricing? + +## 6. Global Context & Risk Factors + +- How do US rates compare to the **ECB**, **Bank of England**, and **Bank of Japan** policy rates? +- What role are **tariff and trade policy** uncertainties playing in Fed deliberations? +- How might **fiscal policy** changes (tax cuts, spending proposals) impact the rate outlook? +- What are the key **upside and downside risks** to the current rate path? + +## 7. Financial Conditions + +- What is the current reading of the **Goldman Sachs Financial Conditions Index** and the **Chicago Fed National Financial Conditions Index**? +- How have **10-year Treasury yields**, **corporate credit spreads**, and **equity valuations** responded to recent policy signals? +- Is the **yield curve** currently inverted, and what does that historically signal? + +Please structure your response with clear headings, use tables for comparative data, include specific numbers and dates, and cite your sources. Flag any data points that may be stale or subject to revision.`, }), payloadType: "application/json", traceId, @@ -478,10 +528,71 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { } // --- Shared prompt content --- - const userMessage = "What is the current Federal Reserve interest rate?"; - const systemPrompt = "You are a helpful financial assistant with access to web search tools."; - const assistantResponse = - "The current Federal Reserve interest rate target range is 4.25% to 4.50%. This was set by the FOMC at their most recent meeting."; + const userMessage = `I need a detailed analysis of the current Federal Reserve interest rate policy, including: + +1. The **current target rate** range +2. How it compares to rates over the past 12 months +3. Any upcoming FOMC meetings and market expectations + +Please include relevant data points and sources where possible.`; + + const systemPrompt = `You are a senior financial analyst AI assistant with expertise in monetary policy, macroeconomics, and Federal Reserve operations. + +## Your Capabilities +- Access to real-time web search for current financial data +- Deep knowledge of FOMC meeting schedules, dot plots, and forward guidance +- Ability to interpret economic indicators (CPI, PCE, employment data) + +## Response Guidelines +- Always cite specific dates and data sources +- Use tables for comparative data when appropriate +- Flag any uncertainty or conflicting signals in the data +- Present information in a structured, scannable format + +## Important Notes +> Never provide investment advice. Present factual analysis only. +> When data is older than 48 hours, note the timestamp explicitly.`; + + const assistantResponse = `## Current Federal Reserve Interest Rate + +The Federal Open Market Committee (FOMC) has set the **federal funds target rate** at a range of **4.25% to 4.50%**, effective since the December 18, 2024 meeting. + +### Rate Trajectory (Past 12 Months) + +| Meeting Date | Decision | Target Range | +|:---|:---|:---| +| Dec 18, 2024 | -25 bps | 4.25% – 4.50% | +| Nov 7, 2024 | -25 bps | 4.50% – 4.75% | +| Sep 18, 2024 | -50 bps | 4.75% – 5.00% | +| Jul 31, 2024 | Hold | 5.25% – 5.50% | +| Jun 12, 2024 | Hold | 5.25% – 5.50% | +| May 1, 2024 | Hold | 5.25% – 5.50% | +| Mar 20, 2024 | Hold | 5.25% – 5.50% | + +### Key Observations + +1. **Easing Cycle Began**: The Fed started cutting rates in September 2024 with an aggressive 50 bps cut, followed by two 25 bps cuts +2. **Cumulative Reduction**: Rates have been lowered by **100 basis points** total from the cycle peak of 5.25%–5.50% +3. **Pause Signals**: Recent FOMC language suggests a *cautious approach* to further cuts, citing: + - Persistent core inflation above the 2% target + - Strong labor market conditions + - Uncertainty around fiscal policy changes + +### Upcoming FOMC Schedule + +The next scheduled meeting is **January 28–29, 2025**. Market expectations via CME FedWatch: + +\`\`\` +Hold (no change): 89.3% +Cut (-25 bps): 10.7% +\`\`\` + +> **Note**: Market pricing strongly favors a hold at the January meeting, with the first expected cut pushed to mid-2025. + +### Sources +- [Federal Reserve Board – Policy Actions](https://federalreserve.gov) +- CME FedWatch Tool (as of Jan 15, 2025) +- FOMC Statement, December 18, 2024`; const toolCallResult = JSON.stringify({ status: 200, contentType: "text/html", @@ -1113,7 +1224,15 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { finishReason: "stop", wrapperDurationMs: 1_400, doDurationMs: 1_200, - responseText: "The document discusses quarterly earnings guidance for tech sector.", + responseText: `### Document Analysis + +The document primarily discusses **quarterly earnings guidance** for the technology sector, with the following key themes: + +- Revenue growth projections of *12–15%* YoY +- Margin compression due to increased R&D spending +- Forward guidance citing \`macroeconomic headwinds\` + +**Confidence**: High (0.92)`, useCompletionStyle: true, providerMetadata: { gateway: { @@ -1215,7 +1334,21 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { finishReason: "stop", wrapperDurationMs: 1_800, doDurationMs: 1_500, - responseText: "The content appears to be a standard financial news article. Classification: SAFE.", + responseText: `## Content Classification Report + +**Category**: Financial News Article +**Risk Level**: SAFE ✓ + +### Analysis Breakdown + +| Criteria | Result | Score | +|:---|:---|---:| +| Factual accuracy | Verified | 0.94 | +| Bias detection | Minimal | 0.12 | +| Misinformation risk | Low | 0.08 | +| Regulatory sensitivity | None detected | 0.02 | + +> This content follows standard financial journalism conventions and references official Federal Reserve communications directly.`, useCompletionStyle: true, providerMetadata: { gateway: { @@ -1312,7 +1445,22 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { finishReason: "stop", wrapperDurationMs: 2_000, doDurationMs: 1_800, - responseText: "Based on the latest FOMC minutes, the committee voted unanimously to maintain rates.", + responseText: `Based on the latest FOMC minutes, the committee voted **unanimously** to maintain rates at the current target range. + +### Key Takeaways from the Minutes + +1. **Labor Market**: Participants noted that employment conditions remain *"solid"* but acknowledged some cooling in job openings +2. **Inflation Outlook**: Core PCE inflation running at 2.8% — still above the 2% target +3. **Forward Guidance**: Several participants emphasized the need for \`patience\` before additional rate adjustments + +#### Notable Quotes + +> "The Committee judges that the risks to achieving its employment and inflation goals are roughly in balance." — *FOMC Statement* + +The next decision point will hinge on incoming data, particularly: +- January CPI release (Feb 12) +- January employment report (Feb 7) +- Q4 GDP second estimate (Feb 27)`, useCompletionStyle: true, providerMetadata: { openrouter: { @@ -1397,7 +1545,27 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { finishReason: "stop", wrapperDurationMs: 4_500, doDurationMs: 4_200, - responseText: "According to the Federal Reserve's most recent announcement on December 18, 2024, the federal funds rate target range was maintained at 4.25% to 4.50%. This decision was made during the December FOMC meeting.", + responseText: `According to the Federal Reserve's most recent announcement on **December 18, 2024**, the federal funds rate target range was maintained at **4.25% to 4.50%**. + +### Context + +This decision was made during the December FOMC meeting, where the committee: + +- Acknowledged *"solid"* economic activity and a labor market that has *"generally eased"* +- Noted inflation remains *"somewhat elevated"* relative to the 2% target +- Projected only **two rate cuts** in 2025 (down from four projected in September) + +### Market Impact + +The announcement triggered a sharp market reaction: + +\`\`\` +S&P 500: -2.95% (largest FOMC-day drop since 2001) +10Y Yield: +11 bps to 4.52% +DXY Index: +1.2% to 108.3 +\`\`\` + +> **Sources**: Federal Reserve Board press release, CME FedWatch, Bloomberg Terminal`, }); // ===================================================================== @@ -1435,8 +1603,34 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { finishReason: "stop", wrapperDurationMs: 12_000, doDurationMs: 11_500, - responseText: "The Federal Reserve has maintained its target range for the federal funds rate at 4.25% to 4.50% since December 2024. This represents a pause in the rate-cutting cycle that began in September 2024. The FOMC has indicated it will continue to assess incoming data, the evolving outlook, and the balance of risks when considering further adjustments.", - responseReasoning: "The user is asking about the current Federal Reserve interest rate. Let me provide a comprehensive answer based on the most recent FOMC decision. I should include context about the rate trajectory and forward guidance.", + responseText: `The Federal Reserve has maintained its target range for the federal funds rate at **4.25% to 4.50%** since December 2024. + +## Rate Cycle Overview + +This represents a **pause** in the rate-cutting cycle that began in September 2024: + +| Phase | Period | Action | +|:---|:---|:---| +| Peak hold | Jul 2023 – Sep 2024 | Held at 5.25%–5.50% | +| Easing begins | Sep 2024 | Cut 50 bps | +| Continued easing | Nov 2024 | Cut 25 bps | +| Final cut (so far) | Dec 2024 | Cut 25 bps | +| Current pause | Jan 2025 – present | Hold | + +### What's Driving the Pause? + +The FOMC has cited three primary factors: + +1. **Sticky inflation**: Core PCE at 2.8% remains above the 2% symmetric target +2. **Resilient growth**: GDP growth of 3.1% in Q3 2024 exceeded expectations +3. **Policy uncertainty**: New administration trade and fiscal policies create *"unusually elevated"* uncertainty + +> The Committee has indicated it will continue to assess incoming data, the evolving outlook, and the balance of risks when considering further adjustments to the target range. + +### Technical Note + +The effective federal funds rate (\`EFFR\`) currently sits at **4.33%**, near the midpoint of the target range. The overnight reverse repo facility (ON RRP) rate is set at **4.25%**.`, + responseReasoning: "The user is asking about the current Federal Reserve interest rate. Let me provide a comprehensive answer based on the most recent FOMC decision. I should include context about the rate trajectory and forward guidance. I'll structure this with a table showing the recent rate changes and explain the pause rationale.", cacheReadTokens: 12400, cacheCreationTokens: 2800, providerMetadata: { @@ -1500,8 +1694,42 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { finishReason: "stop", wrapperDurationMs: 8_000, doDurationMs: 7_500, - responseText: "The Federal Reserve's current target range for the federal funds rate is 4.25% to 4.50%, established at the December 2024 FOMC meeting. The committee has signaled a cautious approach to further rate cuts, citing persistent inflation concerns.", - responseReasoning: "I need to provide accurate, up-to-date information about the Federal Reserve interest rate. The last FOMC meeting was in December 2024 where they held rates steady after three consecutive cuts.", + responseText: `## Federal Funds Rate — Current Status + +The Federal Reserve's current target range is **4.25% to 4.50%**, established at the **December 18, 2024** FOMC meeting. + +### Policy Stance + +The committee has signaled a *cautious approach* to further rate cuts. Key considerations include: + +- **Inflation**: Core PCE remains at 2.8%, above the 2% target +- **Employment**: Unemployment rate stable at 4.2%, with 256K jobs added in December +- **Growth**: Real GDP tracking at ~2.5% annualized + +### Dot Plot Summary (Dec 2024 SEP) + +The median dot plot projections: + +\`\`\`python +# Median FOMC projections +rates = { + "2025": 3.75, # implies 2 cuts of 25bps + "2026": 3.25, # implies 2 additional cuts + "longer_run": 3.00 # neutral rate estimate (up from 2.5%) +} +\`\`\` + +### Risk Assessment + +| Risk Factor | Direction | Magnitude | +|:---|:---:|:---:| +| Tariff-driven inflation | ↑ Upside | Medium | +| Labor market softening | ↓ Downside | Low | +| Fiscal expansion | ↑ Upside | High | +| Global growth slowdown | ↓ Downside | Medium | + +> *"The committee remains attentive to the risks to both sides of its dual mandate."* — Chair Powell, Dec 18 press conference`, + responseReasoning: "I need to provide accurate, up-to-date information about the Federal Reserve interest rate. The last FOMC meeting was in December 2024 where they cut rates by 25 bps after two previous cuts. Let me include the dot plot projections and a risk assessment table for a comprehensive view.", reasoningTokens: 516, providerMetadata: { openai: { @@ -1530,7 +1758,12 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { finishReason: "stop", wrapperDurationMs: 600, doDurationMs: 400, - responseText: "The Federal Reserve rate is currently at 4.25-4.50%.", + responseText: `The Federal Reserve rate is currently at **4.25–4.50%**. + +Key details: +- *Effective date*: December 18, 2024 +- *Next meeting*: January 28–29, 2025 +- *Market expectation*: Hold (\`89.3%\` probability per CME FedWatch)`, }); // ===================================================================== @@ -1547,7 +1780,16 @@ function buildAiSpanTree(params: SpanTreeParams): CreateEventInput[] { finishReason: "stop", wrapperDurationMs: 4_000, doDurationMs: 3_500, - responseText: "Based on the latest FOMC statement, the target rate range remains at 4.25% to 4.50%.", + responseText: `Based on the latest FOMC statement, the target rate range remains at **4.25% to 4.50%**. + +### Additional Context + +The committee's statement included notable language changes: +- Removed reference to *"gaining greater confidence"* on inflation +- Added emphasis on monitoring \`both sides\` of the dual mandate +- Acknowledged *"uncertainty around the economic outlook has increased"* + +Governor Bowman dissented, preferring a **hold** rather than the 25 bps cut — the first governor dissent since 2005.`, }); // ===================================================================== From b5d799472af25ebda5ffd5c67ceb69e377329935 Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Tue, 17 Mar 2026 11:44:34 +0000 Subject: [PATCH 18/30] New model logos --- .../app/assets/icons/AiProviderIcons.tsx | 177 ++++++++++++++++++ .../webapp/app/components/runs/v3/RunIcon.tsx | 39 +++- .../v3/utils/enrichCreatableEvents.server.ts | 88 +++++++-- 3 files changed, 288 insertions(+), 16 deletions(-) create mode 100644 apps/webapp/app/assets/icons/AiProviderIcons.tsx diff --git a/apps/webapp/app/assets/icons/AiProviderIcons.tsx b/apps/webapp/app/assets/icons/AiProviderIcons.tsx new file mode 100644 index 00000000000..85a01b98d63 --- /dev/null +++ b/apps/webapp/app/assets/icons/AiProviderIcons.tsx @@ -0,0 +1,177 @@ +type IconProps = { className?: string }; + +export function OpenAIIcon({ className }: IconProps) { + return ( + + + + ); +} + +export function AnthropicIcon({ className }: IconProps) { + return ( + + + + ); +} + +export function GeminiIcon({ className }: IconProps) { + return ( + + + + ); +} + +export function LlamaIcon({ className }: IconProps) { + return ( + + + + ); +} + +export function DeepseekIcon({ className }: IconProps) { + return ( + + + + + + + + + + + ); +} + +export function XAIIcon({ className }: IconProps) { + return ( + + + + + + + ); +} + +export function PerplexityIcon({ className }: IconProps) { + return ( + + + + ); +} + +export function CerebrasIcon({ className }: IconProps) { + return ( + + + + + + + + ); +} + +export function MistralIcon({ className }: IconProps) { + return ( + + + + + + + + + + + + + ); +} + +export function AzureIcon({ className }: IconProps) { + return ( + + + + ); +} + diff --git a/apps/webapp/app/components/runs/v3/RunIcon.tsx b/apps/webapp/app/components/runs/v3/RunIcon.tsx index 73963ea09b9..572435f75c9 100644 --- a/apps/webapp/app/components/runs/v3/RunIcon.tsx +++ b/apps/webapp/app/components/runs/v3/RunIcon.tsx @@ -3,11 +3,25 @@ import { HandRaisedIcon, InformationCircleIcon, RectangleStackIcon, + SparklesIcon, Squares2X2Icon, TableCellsIcon, TagIcon, + WrenchIcon, } from "@heroicons/react/20/solid"; import { AnthropicLogoIcon } from "~/assets/icons/AnthropicLogoIcon"; +import { + AnthropicIcon, + AzureIcon, + CerebrasIcon, + DeepseekIcon, + GeminiIcon, + LlamaIcon, + MistralIcon, + OpenAIIcon, + PerplexityIcon, + XAIIcon, +} from "~/assets/icons/AiProviderIcons"; import { AttemptIcon } from "~/assets/icons/AttemptIcon"; import { TaskIcon } from "~/assets/icons/TaskIcon"; import { cn } from "~/utils/cn"; @@ -113,8 +127,31 @@ export function RunIcon({ name, className, spanName }: TaskIconProps) { return ; case "streams": return ; + case "hero-sparkles": + return ; + case "hero-wrench": + return ; case "tabler-brand-anthropic": - return ; + case "ai-provider-anthropic": + return ; + case "ai-provider-openai": + return ; + case "ai-provider-gemini": + return ; + case "ai-provider-llama": + return ; + case "ai-provider-deepseek": + return ; + case "ai-provider-xai": + return ; + case "ai-provider-perplexity": + return ; + case "ai-provider-cerebras": + return ; + case "ai-provider-mistral": + return ; + case "ai-provider-azure": + return ; } return ; diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index b9ed86aa874..da6e4605620 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -240,20 +240,16 @@ function enrichStyle(event: CreateEventInput) { return baseStyle; } - // Direct property access and early returns - // GenAI System check const system = props["gen_ai.system"]; - if (typeof system === "string") { - // For gateway/openrouter, derive the icon from the model's provider prefix - // e.g. "mistral/mistral-large-3" → "mistral", "anthropic/claude-..." → "anthropic" - if (system === "gateway" || system === "openrouter") { - const modelId = props["gen_ai.request.model"] ?? props["ai.model.id"]; - if (typeof modelId === "string" && modelId.includes("/")) { - const provider = modelId.split("/")[0].replace(/-/g, ""); - return { ...baseStyle, icon: `tabler-brand-${provider}` }; - } - } - return { ...baseStyle, icon: `tabler-brand-${system.split(".")[0]}` }; + const modelId = props["gen_ai.request.model"] ?? props["ai.model.id"]; + + const provider = resolveAiProvider( + typeof system === "string" ? system : undefined, + typeof modelId === "string" ? modelId : undefined + ); + + if (provider) { + return { ...baseStyle, icon: `ai-provider-${provider}` }; } // Agent workflow check @@ -265,11 +261,11 @@ function enrichStyle(event: CreateEventInput) { const message = event.message; if (typeof message === "string" && message === "ai.toolCall") { - return { ...baseStyle, icon: "tabler-tool" }; + return { ...baseStyle, icon: "hero-wrench" }; } if (typeof message === "string" && message.startsWith("ai.")) { - return { ...baseStyle, icon: "tabler-sparkles" }; + return { ...baseStyle, icon: "hero-sparkles" }; } return baseStyle; @@ -358,3 +354,65 @@ function formatPythonStyle(template: string, values: Record): strin return hasRepr ? repr(value) : String(value); }); } + +type AiProvider = + | "anthropic" + | "openai" + | "gemini" + | "llama" + | "deepseek" + | "xai" + | "perplexity" + | "cerebras" + | "azure" + | "mistral"; + +const systemToProvider: Record = { + anthropic: "anthropic", + openai: "openai", + azure: "azure", + "google.generative-ai": "gemini", + google: "gemini", + xai: "xai", + deepseek: "deepseek", + cerebras: "cerebras", + perplexity: "perplexity", + "meta-llama": "llama", + mistral: "mistral", +}; + +const modelPatterns: [RegExp, AiProvider][] = [ + [/\banthropic\b|claude/i, "anthropic"], + [/\bopenai\b|gpt-|o[134]-|chatgpt/i, "openai"], + [/gemini/i, "gemini"], + [/llama/i, "llama"], + [/deepseek/i, "deepseek"], + [/grok/i, "xai"], + [/sonar/i, "perplexity"], + [/cerebras/i, "cerebras"], + [/mistral|mixtral|codestral|pixtral/i, "mistral"], +]; + +function resolveAiProvider( + system: string | undefined, + modelId: string | undefined +): AiProvider | undefined { + if (modelId) { + if (modelId.includes("/")) { + const prefix = modelId.split("/")[0].toLowerCase(); + const fromPrefix = systemToProvider[prefix]; + if (fromPrefix) return fromPrefix; + } + + for (const [pattern, provider] of modelPatterns) { + if (pattern.test(modelId)) return provider; + } + } + + if (system) { + const normalized = system.toLowerCase().split(".")[0]; + return systemToProvider[system] ?? systemToProvider[normalized]; + } + + return undefined; +} From 3a49af8e952d6dcf333cfa2f8a966fc080de9b3b Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Tue, 17 Mar 2026 11:44:53 +0000 Subject: [PATCH 19/30] Default wider side panel to better view markdown --- .../route.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx index e02d29b95b5..118226906e1 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx @@ -119,7 +119,7 @@ const resizableSettings = { }, inspector: { id: "inspector", - default: "430px" as const, + default: "500px" as const, min: "50px" as const, }, }, From c322ee7779e72fad6df3cb44b7d4aede86b29f35 Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Tue, 17 Mar 2026 12:09:47 +0000 Subject: [PATCH 20/30] Render JSON object output in our code block component rather than as a string --- .../components/runs/v3/ai/AIChatMessages.tsx | 27 ++++++++++++ .../components/runs/v3/ai/AISpanDetails.tsx | 44 ++++++++++++++----- .../runs/v3/ai/extractAISpanData.ts | 3 +- .../webapp/app/components/runs/v3/ai/types.ts | 2 + 4 files changed, 64 insertions(+), 12 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx index a50ef8ae806..7ef9d574cee 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx @@ -109,6 +109,17 @@ function UserSection({ text }: { text: string }) { // Assistant response (with markdown/raw toggle) // --------------------------------------------------------------------------- +function isJsonString(value: string): boolean { + const trimmed = value.trimStart(); + if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return false; + try { + JSON.parse(value); + return true; + } catch { + return false; + } +} + export function AssistantResponse({ text, headerLabel = "Assistant", @@ -116,6 +127,7 @@ export function AssistantResponse({ text: string; headerLabel?: string; }) { + const isJson = isJsonString(text); const [mode, setMode] = useState<"rendered" | "raw">("rendered"); const [copied, setCopied] = useState(false); @@ -125,6 +137,21 @@ export function AssistantResponse({ setTimeout(() => setCopied(false), 2000); } + if (isJson) { + return ( +
+ + +
+ ); + } + return (
- {/* Tags + Stats */} - {/* Input (last user prompt) */} {userText && (
Input @@ -94,9 +93,20 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) {
)} - {/* Output (assistant response or tool calls) */} {outputText && } - {outputToolNames.length > 0 && !outputText && ( + {!outputText && outputObject && ( +
+ Output + +
+ )} + {outputToolNames.length > 0 && !outputText && !outputObject && (
Output @@ -112,12 +122,26 @@ function OverviewTab({ aiData }: { aiData: AISpanData }) { } function MessagesTab({ aiData }: { aiData: AISpanData }) { + const showFallbackText = aiData.responseText && !hasAssistantItem(aiData.items); + const showFallbackObject = + !showFallbackText && aiData.responseObject && !hasAssistantItem(aiData.items); + return (
{aiData.items && aiData.items.length > 0 && } - {aiData.responseText && !hasAssistantItem(aiData.items) && ( - + {showFallbackText && } + {showFallbackObject && ( +
+ Assistant + +
)}
@@ -158,6 +182,7 @@ function CopyRawFooter({ rawProperties }: { rawProperties: string }) { function extractInputOutput(aiData: AISpanData): { userText: string | undefined; outputText: string | undefined; + outputObject: string | undefined; outputToolNames: string[]; } { let userText: string | undefined; @@ -165,7 +190,6 @@ function extractInputOutput(aiData: AISpanData): { const outputToolNames: string[] = []; if (aiData.items) { - // Find the last user message for (let i = aiData.items.length - 1; i >= 0; i--) { if (aiData.items[i].type === "user") { userText = (aiData.items[i] as { type: "user"; text: string }).text; @@ -173,7 +197,6 @@ function extractInputOutput(aiData: AISpanData): { } } - // Find the last assistant or tool-use item as the output for (let i = aiData.items.length - 1; i >= 0; i--) { const item = aiData.items[i]; if (item.type === "assistant") { @@ -189,12 +212,11 @@ function extractInputOutput(aiData: AISpanData): { } } - // Fall back to responseText if no assistant item found if (!outputText && aiData.responseText) { outputText = aiData.responseText; } - return { userText, outputText, outputToolNames }; + return { userText, outputText, outputObject: aiData.responseObject, outputToolNames }; } function hasAssistantItem(items: DisplayItem[] | undefined): boolean { diff --git a/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts b/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts index 82c1014d5ab..dbe9b61d952 100644 --- a/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts +++ b/apps/webapp/app/components/runs/v3/ai/extractAISpanData.ts @@ -77,7 +77,8 @@ export function extractAISpanData( inputCost: num(triggerLlm.input_cost), outputCost: num(triggerLlm.output_cost), totalCost: num(triggerLlm.total_cost), - responseText: str(aiResponse.text) || str(aiResponse.object) || undefined, + responseText: str(aiResponse.text) || undefined, + responseObject: str(aiResponse.object) || undefined, toolDefinitions: toolDefs, items: buildDisplayItems(aiPrompt.messages, aiResponse.toolCalls, toolDefs), }; diff --git a/apps/webapp/app/components/runs/v3/ai/types.ts b/apps/webapp/app/components/runs/v3/ai/types.ts index 70d75533de2..be10ee9bcd1 100644 --- a/apps/webapp/app/components/runs/v3/ai/types.ts +++ b/apps/webapp/app/components/runs/v3/ai/types.ts @@ -94,6 +94,8 @@ export type AISpanData = { // Response text (final assistant output) responseText?: string; + // Structured object response (JSON) — mutually exclusive with responseText + responseObject?: string; // Tool definitions (from ai.prompt.tools) toolDefinitions?: ToolDefinition[]; From a262c7aa789fba8d50c6c2a8785bf4c1bcea3e2d Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Tue, 17 Mar 2026 12:16:08 +0000 Subject: [PATCH 21/30] Use streamdown for system prompts --- .../app/components/runs/v3/ai/AIChatMessages.tsx | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx index 7ef9d574cee..02ab46eb5f4 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx @@ -65,6 +65,7 @@ function SystemSection({ text }: { text: string }) { const [expanded, setExpanded] = useState(false); const isLong = text.length > 150; const preview = isLong ? text.slice(0, 150) + "..." : text; + const displayText = expanded || !isLong ? text : preview; return (
@@ -82,8 +83,10 @@ function SystemSection({ text }: { text: string }) { } /> - - {expanded || !isLong ? text : preview} + + {displayText}}> + {displayText} +
@@ -99,7 +102,11 @@ function UserSection({ text }: { text: string }) {
- {text} + + {text}}> + {text} + +
); From e372fa0cb5b4c5b3510adc5de535534113cf37dd Mon Sep 17 00:00:00 2001 From: James Ritchie Date: Tue, 17 Mar 2026 12:27:16 +0000 Subject: [PATCH 22/30] =?UTF-8?q?toolCall=20doesn=E2=80=99t=20get=20wrappe?= =?UTF-8?q?d=20in=20chat=20div?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/components/runs/v3/ai/AIChatMessages.tsx | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx index 02ab46eb5f4..9e5a799e1bc 100644 --- a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx +++ b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx @@ -212,13 +212,11 @@ function ToolUseSection({ tools }: { tools: ToolUse[] }) { return (
- -
- {tools.map((tool) => ( - - ))} -
- +
+ {tools.map((tool) => ( + + ))} +
); } From e4ae511ab066ec9ca63f449c947bb007278395a6 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 17 Mar 2026 14:10:54 +0000 Subject: [PATCH 23/30] fixed failing tests because of icon name changes --- apps/webapp/test/otlpExporter.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/webapp/test/otlpExporter.test.ts b/apps/webapp/test/otlpExporter.test.ts index 0194ae6bf74..f07019b4a25 100644 --- a/apps/webapp/test/otlpExporter.test.ts +++ b/apps/webapp/test/otlpExporter.test.ts @@ -87,7 +87,7 @@ describe("OTLPExporter", () => { const event = $events[0]; expect(event.message).toBe("Responses API with 'gpt-4o'"); expect(event.style).toMatchObject({ - icon: "tabler-brand-openai", + icon: "ai-provider-openai", }); }); @@ -165,7 +165,7 @@ describe("OTLPExporter", () => { const event = $events[0]; expect(event.message).toBe("Responses API with gpt-4o"); expect(event.style).toMatchObject({ - icon: "tabler-brand-openai", + icon: "ai-provider-openai", }); // Enrichment also adds model/token pills as accessories const style = event.style as Record; @@ -230,7 +230,7 @@ describe("OTLPExporter", () => { const $events = enrichCreatableEvents(events); expect($events[0].message).toBe("Using 'gpt-4' with temperature 0.7"); expect($events[0].style).toEqual({ - icon: "tabler-brand-openai", + icon: "ai-provider-openai", }); }); @@ -261,7 +261,7 @@ describe("OTLPExporter", () => { const $events = enrichCreatableEvents(events); expect($events[0].message).toBe("Count is 42 and enabled is true"); expect($events[0].style).toEqual({ - icon: "tabler-brand-anthropic", + icon: "ai-provider-anthropic", }); }); @@ -290,7 +290,7 @@ describe("OTLPExporter", () => { const $events = enrichCreatableEvents(events); expect($events[0].message).toBe("Plain message without variables"); expect($events[0].style).toEqual({ - icon: "tabler-brand-openai", + icon: "ai-provider-openai", }); }); @@ -346,7 +346,7 @@ describe("OTLPExporter", () => { const $events = enrichCreatableEvents(events); expect($events[0].style).toEqual({ existingStyle: "value", - icon: "tabler-brand-openai", // GenAI enricher wins because it's first + icon: "ai-provider-openai", // GenAI enricher wins because it's first }); }); From d5a32be360dbf37ad965c62e09e32f9d671af581 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 17 Mar 2026 14:12:07 +0000 Subject: [PATCH 24/30] strip crumbs --- apps/webapp/app/v3/otlpExporter.server.ts | 18 --------------- apps/webapp/seed-ai-spans.mts | 28 ----------------------- 2 files changed, 46 deletions(-) diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index 0a86fb65ec9..1bc03b7621e 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -38,8 +38,6 @@ import type { import { startSpan } from "./tracing.server"; import { enrichCreatableEvents } from "./utils/enrichCreatableEvents.server"; import "./llmPricingRegistry.server"; // Initialize LLM pricing registry on startup -import { trail } from "agentcrumbs"; // @crumbs -const crumbOtlp = trail("webapp:otlp-exporter"); // @crumbs import { env } from "~/env.server"; import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings"; import { singleton } from "~/utils/singleton"; @@ -396,22 +394,6 @@ function convertSpansToCreateableEvents( const runTags = extractArrayAttribute(span.attributes ?? [], SemanticInternalAttributes.RUN_TAGS); - // #region @crumbs - if (span.attributes) { - crumbOtlp("span raw OTEL attrs", { - spanName: span.name, - spanId: binaryToHex(span.spanId), - attrCount: span.attributes.length, - attrs: span.attributes.map((a) => ({ - key: a.key, - type: a.value?.stringValue !== undefined ? "string" : a.value?.intValue !== undefined ? "int" : a.value?.doubleValue !== undefined ? "double" : a.value?.boolValue !== undefined ? "bool" : a.value?.arrayValue ? "array" : a.value?.bytesValue ? "bytes" : "unknown", - ...(a.value?.arrayValue ? { arrayLen: a.value.arrayValue.values?.length } : {}), - ...(a.value?.stringValue !== undefined ? { strLen: a.value.stringValue.length } : {}), - })), - }); - } - // #endregion @crumbs - const properties = truncateAttributes( convertKeyValueItemsToMap(span.attributes ?? [], [], undefined, [ diff --git a/apps/webapp/seed-ai-spans.mts b/apps/webapp/seed-ai-spans.mts index cc701843d31..35ec3d5851e 100644 --- a/apps/webapp/seed-ai-spans.mts +++ b/apps/webapp/seed-ai-spans.mts @@ -1,5 +1,3 @@ -import { trail } from "agentcrumbs"; // @crumbs -const crumb = trail("webapp"); // @crumbs import { prisma } from "./app/db.server"; import { createOrganization } from "./app/models/organization.server"; import { createProject } from "./app/models/project.server"; @@ -169,11 +167,9 @@ function eventToLlmMetricsRow(event: CreateEventInput): LlmMetricsV1Input { async function seedAiSpans() { const seedUserId = randomUserId(); - crumb("seed started"); // @crumbs console.log(`Starting AI span seed (userId: ${seedUserId})...\n`); // 1. Find user - crumb("finding user"); // @crumbs const user = await prisma.user.findUnique({ where: { email: "local@trigger.dev" }, }); @@ -181,10 +177,8 @@ async function seedAiSpans() { console.error("User local@trigger.dev not found. Run `pnpm run db:seed` first."); process.exit(1); } - crumb("user found", { userId: user.id }); // @crumbs // 2. Find or create org - crumb("finding/creating org"); // @crumbs let org = await prisma.organization.findFirst({ where: { title: ORG_TITLE, members: { some: { userId: user.id } } }, }); @@ -194,10 +188,8 @@ async function seedAiSpans() { } else { console.log(`Org exists: ${org.title} (${org.slug})`); } - crumb("org ready", { orgId: org.id, slug: org.slug }); // @crumbs // 3. Find or create project - crumb("finding/creating project"); // @crumbs let project = await prisma.project.findFirst({ where: { name: PROJECT_NAME, organizationId: org.id }, }); @@ -212,10 +204,8 @@ async function seedAiSpans() { } else { console.log(`Project exists: ${project.name} (${project.externalRef})`); } - crumb("project ready", { projectId: project.id, ref: project.externalRef }); // @crumbs // 4. Get DEVELOPMENT environment - crumb("finding dev environment"); // @crumbs const runtimeEnv = await prisma.runtimeEnvironment.findFirst({ where: { projectId: project.id, type: "DEVELOPMENT" }, }); @@ -223,10 +213,8 @@ async function seedAiSpans() { console.error("No DEVELOPMENT environment found for project."); process.exit(1); } - crumb("dev env found", { envId: runtimeEnv.id }); // @crumbs // 5. Upsert background worker - crumb("upserting worker/task/queue"); // @crumbs const worker = await prisma.backgroundWorker.upsert({ where: { projectId_runtimeEnvironmentId_version: { @@ -281,10 +269,7 @@ async function seedAiSpans() { }, }); - crumb("infra upserts done"); // @crumbs - // 8. Create the TaskRun - crumb("creating TaskRun"); // @crumbs const traceId = generateTraceId(); const rootSpanId = generateSpanId(); const now = Date.now(); @@ -367,11 +352,9 @@ Please structure your response with clear headings, use tables for comparative d }, }); - crumb("TaskRun created", { runId: run.friendlyId, traceId }); // @crumbs console.log(`Created TaskRun: ${run.friendlyId}`); // 9. Build span tree - crumb("building span tree"); // @crumbs const events = buildAiSpanTree({ traceId, rootSpanId, @@ -384,22 +367,18 @@ Please structure your response with clear headings, use tables for comparative d seedUserId, }); - crumb("span tree built", { spanCount: events.length }); // @crumbs console.log(`Built ${events.length} spans`); // 10. Seed LLM pricing and enrich - crumb("seeding LLM pricing"); // @crumbs const seedResult = await seedLlmPricing(prisma); console.log( `LLM pricing: ${seedResult.modelsCreated} created, ${seedResult.modelsSkipped} skipped` ); - crumb("loading pricing registry"); // @crumbs const registry = new ModelPricingRegistry(prisma); setLlmPricingRegistry(registry); await registry.loadFromDatabase(); - crumb("enriching events"); // @crumbs const enriched = enrichCreatableEvents(events); const enrichedCount = enriched.filter((e) => e._llmMetrics != null).length; @@ -408,10 +387,7 @@ Please structure your response with clear headings, use tables for comparative d `Enriched ${enrichedCount} spans with LLM cost (total: $${totalCost.toFixed(6)})` ); - crumb("enrichment done", { enrichedCount, totalCost }); // @crumbs - // 11. Insert into ClickHouse - crumb("inserting into ClickHouse"); // @crumbs const clickhouseUrl = process.env.CLICKHOUSE_URL ?? process.env.EVENTS_CLICKHOUSE_URL; if (!clickhouseUrl) { console.error("CLICKHOUSE_URL or EVENTS_CLICKHOUSE_URL not set"); @@ -426,13 +402,10 @@ Please structure your response with clear headings, use tables for comparative d const chRows = enriched.map(eventToClickhouseRow); await clickhouse.taskEventsV2.insert(chRows); - crumb("task events inserted", { rowCount: chRows.length }); // @crumbs - // Insert LLM usage rows const llmRows = enriched.filter((e) => e._llmMetrics != null).map(eventToLlmMetricsRow); if (llmRows.length > 0) { await clickhouse.llmMetrics.insert(llmRows); - crumb("llm metrics inserted", { rowCount: llmRows.length }); // @crumbs } // 12. Output @@ -443,7 +416,6 @@ Please structure your response with clear headings, use tables for comparative d console.log(`Spans: ${events.length}`); console.log(`LLM cost enriched: ${enrichedCount}`); console.log(`Total cost: $${totalCost.toFixed(6)}`); - crumb("seed complete"); // @crumbs process.exit(0); } From 9e2b5781531523d86e192f9e5499c0eaad1830b0 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 17 Mar 2026 15:34:52 +0000 Subject: [PATCH 25/30] give the llm model registry a short window to become ready when the server first tries to enrich spans so we don't miss enriching the first Xms of spans every time a server boots --- apps/webapp/app/env.server.ts | 1 + .../app/v3/llmPricingRegistry.server.ts | 38 ++++++++++++++----- apps/webapp/app/v3/otlpExporter.server.ts | 3 +- .../v3/utils/enrichCreatableEvents.server.ts | 5 ++- internal-packages/llm-pricing/src/registry.ts | 13 ++++++- 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 05d8f93f66c..0d3149e392a 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1287,6 +1287,7 @@ const EnvironmentSchema = z LLM_COST_TRACKING_ENABLED: BoolEnv.default(true), LLM_PRICING_RELOAD_INTERVAL_MS: z.coerce.number().int().default(5 * 60 * 1000), // 5 minutes LLM_PRICING_SEED_ON_STARTUP: BoolEnv.default(false), + LLM_PRICING_READY_TIMEOUT_MS: z.coerce.number().int().default(500), // Bootstrap TRIGGER_BOOTSTRAP_ENABLED: z.string().default("0"), diff --git a/apps/webapp/app/v3/llmPricingRegistry.server.ts b/apps/webapp/app/v3/llmPricingRegistry.server.ts index e90a84689f5..627609bb1d8 100644 --- a/apps/webapp/app/v3/llmPricingRegistry.server.ts +++ b/apps/webapp/app/v3/llmPricingRegistry.server.ts @@ -1,12 +1,13 @@ import { ModelPricingRegistry, seedLlmPricing } from "@internal/llm-pricing"; import { prisma, $replica } from "~/db.server"; import { env } from "~/env.server"; +import { signalsEmitter } from "~/services/signals.server"; import { singleton } from "~/utils/singleton"; import { setLlmPricingRegistry } from "./utils/enrichCreatableEvents.server"; async function initRegistry(registry: ModelPricingRegistry) { if (env.LLM_PRICING_SEED_ON_STARTUP) { - const result = await seedLlmPricing(prisma); + await seedLlmPricing(prisma); } await registry.loadFromDatabase(); @@ -28,15 +29,34 @@ export const llmPricingRegistry = singleton("llmPricingRegistry", () => { // Periodic reload const reloadInterval = env.LLM_PRICING_RELOAD_INTERVAL_MS; - setInterval(() => { - registry - .reload() - .then(() => { - }) - .catch((err) => { - console.error("Failed to reload LLM pricing registry", err); - }); + const interval = setInterval(() => { + registry.reload().catch((err) => { + console.error("Failed to reload LLM pricing registry", err); + }); }, reloadInterval); + signalsEmitter.on("SIGTERM", () => { + clearInterval(interval); + }); + signalsEmitter.on("SIGINT", () => { + clearInterval(interval); + }); + return registry; }); + +/** + * Wait for the LLM pricing registry to finish its initial load, with a timeout. + * After the first call resolves (or times out), subsequent calls are no-ops. + */ +export async function waitForLlmPricingReady(): Promise { + if (!llmPricingRegistry || llmPricingRegistry.isLoaded) return; + + const timeoutMs = env.LLM_PRICING_READY_TIMEOUT_MS; + if (timeoutMs <= 0) return; + + await Promise.race([ + llmPricingRegistry.isReady, + new Promise((resolve) => setTimeout(resolve, timeoutMs)), + ]); +} diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index 1bc03b7621e..5fe2624557d 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -37,7 +37,7 @@ import type { } from "./eventRepository/eventRepository.types"; import { startSpan } from "./tracing.server"; import { enrichCreatableEvents } from "./utils/enrichCreatableEvents.server"; -import "./llmPricingRegistry.server"; // Initialize LLM pricing registry on startup +import { waitForLlmPricingReady } from "./llmPricingRegistry.server"; import { env } from "~/env.server"; import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings"; import { singleton } from "~/utils/singleton"; @@ -129,6 +129,7 @@ class OTLPExporter { for (const [store, events] of Object.entries(eventsGroupedByStore)) { const eventRepository = this.#getEventRepositoryForStore(store); + await waitForLlmPricingReady(); const enrichedEvents = enrichCreatableEvents(events); this.#logEventsVerbose(enrichedEvents, `exportEvents ${store}`); diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index da6e4605620..1896b11f051 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -20,6 +20,8 @@ type CostRegistry = { let _registry: CostRegistry | undefined; +const ENRICHABLE_KINDS = new Set(["INTERNAL", "SERVER", "CLIENT", "CONSUMER", "PRODUCER"]); + export function setLlmPricingRegistry(registry: CostRegistry): void { _registry = registry; } @@ -46,8 +48,7 @@ function enrichLlmMetrics(event: CreateEventInput): void { if (!props) return; // Only enrich span-like events (INTERNAL, SERVER, CLIENT, CONSUMER, PRODUCER — not LOG, UNSPECIFIED) - const enrichableKinds = new Set(["INTERNAL", "SERVER", "CLIENT", "CONSUMER", "PRODUCER"]); - if (!enrichableKinds.has(event.kind as string)) return; + if (!ENRICHABLE_KINDS.has(event.kind as string)) return; // Skip partial spans (they don't have final token counts) if (event.isPartial) return; diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts index e1faaaa169f..653b473b626 100644 --- a/internal-packages/llm-pricing/src/registry.ts +++ b/internal-packages/llm-pricing/src/registry.ts @@ -24,9 +24,16 @@ export class ModelPricingRegistry { private _patterns: CompiledPattern[] = []; private _exactMatchCache: Map = new Map(); private _loaded = false; + private _readyResolve!: () => void; + + /** Resolves once the initial `loadFromDatabase()` completes successfully. */ + readonly isReady: Promise; constructor(prisma: PrismaClient | PrismaReplicaClient) { this._prisma = prisma; + this.isReady = new Promise((resolve) => { + this._readyResolve = resolve; + }); } get isLoaded(): boolean { @@ -81,7 +88,11 @@ export class ModelPricingRegistry { this._patterns = compiled; this._exactMatchCache.clear(); - this._loaded = true; + + if (!this._loaded) { + this._loaded = true; + this._readyResolve(); + } } async reload(): Promise { From c7d2706b79f95184b917a0076d26c95e9e087353 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 17 Mar 2026 15:56:32 +0000 Subject: [PATCH 26/30] A few comments --- .../v3/eventRepository/clickhouseEventRepository.server.ts | 5 +++++ internal-packages/llm-pricing/src/registry.ts | 3 +++ 2 files changed, 8 insertions(+) diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts index 86f3560669c..54586666388 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts @@ -1379,6 +1379,11 @@ export class ClickhouseEventRepository implements IEventRepository { } } + // Parse attributes from the first record that has them, then re-parse for the + // completed SPAN record. The completed record's attributes are a superset of the + // partial's (includes enriched trigger.llm.* cost data added during ingestion). + // This means at most 2x JSON.parse per span detail query, but only on this + // read path (span detail view), not on ingestion. if (typeof record.attributes_text === "string") { const shouldUpdate = span.properties == null || diff --git a/internal-packages/llm-pricing/src/registry.ts b/internal-packages/llm-pricing/src/registry.ts index 653b473b626..80da40ba980 100644 --- a/internal-packages/llm-pricing/src/registry.ts +++ b/internal-packages/llm-pricing/src/registry.ts @@ -22,6 +22,9 @@ function compilePattern(pattern: string): RegExp { export class ModelPricingRegistry { private _prisma: PrismaClient | PrismaReplicaClient; private _patterns: CompiledPattern[] = []; + // TODO: When we add project-based models (users adding their own), this cache grows unbounded + // between reloads. Fine-tuned model IDs (e.g. "ft:gpt-3.5-turbo:org:name:id") create unique + // entries per model string. Consider adding an LRU cap or size limit at that point. private _exactMatchCache: Map = new Map(); private _loaded = false; private _readyResolve!: () => void; From 6e230f511e1cbe76cf6540a22a9848ee96c34203 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 17 Mar 2026 16:03:29 +0000 Subject: [PATCH 27/30] Add env vars for configuring the llm_metrics_v1 flush scheudler settings --- apps/webapp/app/env.server.ts | 4 ++++ .../clickhouseEventRepository.server.ts | 15 ++++++++++----- .../clickhouseEventRepositoryInstance.server.ts | 8 ++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 0d3149e392a..2e6da79fdf1 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1288,6 +1288,10 @@ const EnvironmentSchema = z LLM_PRICING_RELOAD_INTERVAL_MS: z.coerce.number().int().default(5 * 60 * 1000), // 5 minutes LLM_PRICING_SEED_ON_STARTUP: BoolEnv.default(false), LLM_PRICING_READY_TIMEOUT_MS: z.coerce.number().int().default(500), + LLM_METRICS_BATCH_SIZE: z.coerce.number().int().default(5000), + LLM_METRICS_FLUSH_INTERVAL_MS: z.coerce.number().int().default(2000), + LLM_METRICS_MAX_BATCH_SIZE: z.coerce.number().int().default(10000), + LLM_METRICS_MAX_CONCURRENCY: z.coerce.number().int().default(2), // Bootstrap TRIGGER_BOOTSTRAP_ENABLED: z.string().default("0"), diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts index 54586666388..de6d77835d0 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts @@ -86,6 +86,11 @@ export type ClickhouseEventRepositoryConfig = { * - "v2": Uses task_events_v2 (partitioned by inserted_at to avoid "too many parts" errors) */ version?: "v1" | "v2"; + /** LLM metrics flush scheduler config */ + llmMetricsBatchSize?: number; + llmMetricsFlushInterval?: number; + llmMetricsMaxBatchSize?: number; + llmMetricsMaxConcurrency?: number; }; /** @@ -123,13 +128,13 @@ export class ClickhouseEventRepository implements IEventRepository { }); this._llmMetricsFlushScheduler = new DynamicFlushScheduler({ - batchSize: 5000, - flushInterval: 2000, + batchSize: config.llmMetricsBatchSize ?? 5000, + flushInterval: config.llmMetricsFlushInterval ?? 2000, callback: this.#flushLlmMetricsBatch.bind(this), minConcurrency: 1, - maxConcurrency: 2, - maxBatchSize: 10000, - memoryPressureThreshold: 10000, + maxConcurrency: config.llmMetricsMaxConcurrency ?? 2, + maxBatchSize: config.llmMetricsMaxBatchSize ?? 10000, + memoryPressureThreshold: config.llmMetricsMaxBatchSize ?? 10000, loadSheddingEnabled: false, }); } diff --git a/apps/webapp/app/v3/eventRepository/clickhouseEventRepositoryInstance.server.ts b/apps/webapp/app/v3/eventRepository/clickhouseEventRepositoryInstance.server.ts index 36fb13c6e96..d4e28c5841a 100644 --- a/apps/webapp/app/v3/eventRepository/clickhouseEventRepositoryInstance.server.ts +++ b/apps/webapp/app/v3/eventRepository/clickhouseEventRepositoryInstance.server.ts @@ -64,6 +64,10 @@ function initializeClickhouseRepository() { asyncInsertMaxDataSize: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE, asyncInsertBusyTimeoutMs: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS, startTimeMaxAgeMs: env.EVENTS_CLICKHOUSE_START_TIME_MAX_AGE_MS, + llmMetricsBatchSize: env.LLM_METRICS_BATCH_SIZE, + llmMetricsFlushInterval: env.LLM_METRICS_FLUSH_INTERVAL_MS, + llmMetricsMaxBatchSize: env.LLM_METRICS_MAX_BATCH_SIZE, + llmMetricsMaxConcurrency: env.LLM_METRICS_MAX_CONCURRENCY, version: "v1", }); @@ -97,6 +101,10 @@ function initializeClickhouseRepositoryV2() { waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1", asyncInsertMaxDataSize: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE, asyncInsertBusyTimeoutMs: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS, + llmMetricsBatchSize: env.LLM_METRICS_BATCH_SIZE, + llmMetricsFlushInterval: env.LLM_METRICS_FLUSH_INTERVAL_MS, + llmMetricsMaxBatchSize: env.LLM_METRICS_MAX_BATCH_SIZE, + llmMetricsMaxConcurrency: env.LLM_METRICS_MAX_CONCURRENCY, version: "v2", }); From 430c0e0d29535155ceb15bf4e3c3f4fce2cb3688 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 17 Mar 2026 16:34:24 +0000 Subject: [PATCH 28/30] as string bad --- .../app/v3/utils/enrichCreatableEvents.server.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts index 1896b11f051..06458c396f9 100644 --- a/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts +++ b/apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts @@ -163,7 +163,11 @@ function enrichLlmMetrics(event: CreateEventInput): void { : ""; const operationId = typeof props["ai.operationId"] === "string" ? props["ai.operationId"] - : (props["gen_ai.operation.name"] as string) ?? (props["operation.name"] as string) ?? ""; + : typeof props["gen_ai.operation.name"] === "string" + ? props["gen_ai.operation.name"] + : typeof props["operation.name"] === "string" + ? props["operation.name"] + : ""; const msToFirstChunk = typeof props["ai.response.msToFirstChunk"] === "number" ? props["ai.response.msToFirstChunk"] : 0; @@ -175,8 +179,8 @@ function enrichLlmMetrics(event: CreateEventInput): void { // Set _llmMetrics side-channel for dual-write to llm_metrics_v1 const llmMetrics: LlmMetricsData = { - genAiSystem: (props["gen_ai.system"] as string) ?? "unknown", - requestModel: (props["gen_ai.request.model"] as string) ?? responseModel, + genAiSystem: typeof props["gen_ai.system"] === "string" ? props["gen_ai.system"] : "unknown", + requestModel: typeof props["gen_ai.request.model"] === "string" ? props["gen_ai.request.model"] : responseModel, responseModel, matchedModelId: cost?.matchedModelId ?? "", operationId, From 0ea16d4f8f87a5b24de8a82ac20205201f84384a Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 17 Mar 2026 16:36:07 +0000 Subject: [PATCH 29/30] register default model prices --- .../llm-pricing/src/default-model-prices.json | 25 +++++++++++++++++++ .../llm-pricing/src/defaultPrices.ts | 20 +++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/internal-packages/llm-pricing/src/default-model-prices.json b/internal-packages/llm-pricing/src/default-model-prices.json index 486c6c512b5..4d2394ec082 100644 --- a/internal-packages/llm-pricing/src/default-model-prices.json +++ b/internal-packages/llm-pricing/src/default-model-prices.json @@ -3834,5 +3834,30 @@ } } ] + }, + { + "id": "029e6695-ff24-47f0-b37b-7285fb2e5785", + "modelName": "gemini-live-2.5-flash-native-audio", + "matchPattern": "(?i)^(google\/)?(gemini-live-2.5-flash-native-audio)$", + "createdAt": "2026-03-16T00:00:00.000Z", + "updatedAt": "2026-03-16T00:00:00.000Z", + "tokenizerConfig": null, + "tokenizerId": null, + "pricingTiers": [ + { + "id": "029e6695-ff24-47f0-b37b-7285fb2e5785_tier_default", + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text": 0.5e-6, + "input_audio": 3e-6, + "input_image": 3e-6, + "output_text": 2e-6, + "output_audio": 12e-6 + } + } + ] } ] diff --git a/internal-packages/llm-pricing/src/defaultPrices.ts b/internal-packages/llm-pricing/src/defaultPrices.ts index 689944a6432..87013e55337 100644 --- a/internal-packages/llm-pricing/src/defaultPrices.ts +++ b/internal-packages/llm-pricing/src/defaultPrices.ts @@ -2980,5 +2980,25 @@ export const defaultModelPrices: DefaultModelDefinition[] = [ } } ] + }, + { + "modelName": "gemini-live-2.5-flash-native-audio", + "matchPattern": "(?i)^(google/)?(gemini-live-2.5-flash-native-audio)$", + "startDate": "2026-03-16T00:00:00.000Z", + "pricingTiers": [ + { + "name": "Standard", + "isDefault": true, + "priority": 0, + "conditions": [], + "prices": { + "input_text": 5e-7, + "input_audio": 0.000003, + "input_image": 0.000003, + "output_text": 0.000002, + "output_audio": 0.000012 + } + } + ] } ]; From 2af8a1bf3f4b4ba5ea8113e768a960031dcce4ff Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 17 Mar 2026 16:44:38 +0000 Subject: [PATCH 30/30] add llm_metrics to the get_query_schema tool desc --- packages/cli-v3/src/mcp/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli-v3/src/mcp/config.ts b/packages/cli-v3/src/mcp/config.ts index c4532b27f5e..c3615420ad5 100644 --- a/packages/cli-v3/src/mcp/config.ts +++ b/packages/cli-v3/src/mcp/config.ts @@ -116,7 +116,7 @@ export const toolsMetadata = { name: "get_query_schema", title: "Get Query Schema", description: - "Get the column schema for a specific TRQL table. Available tables: 'runs' (task execution data), 'metrics' (CPU, memory, custom metrics). Returns columns, types, descriptions, and allowed values for the specified table.", + "Get the column schema for a specific TRQL table. Available tables: 'runs' (task execution data), 'metrics' (CPU, memory, custom metrics), 'llm_metrics' (LLM token usage, costs, latency). Returns columns, types, descriptions, and allowed values for the specified table.", }, list_dashboards: { name: "list_dashboards",