@@ -59,6 +70,21 @@ function SpanAccessory({
}
}
+function SpanPill({ text, icon }: { text: string; icon?: string }) {
+ const hasIcon = icon && tablerIcons.has(icon);
+
+ return (
+
+ {hasIcon && (
+
+ )}
+ {text}
+
+ );
+}
+
export function SpanCodePathAccessory({
accessory,
className,
diff --git a/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx
new file mode 100644
index 00000000000..9e5a799e1bc
--- /dev/null
+++ b/apps/webapp/app/components/runs/v3/ai/AIChatMessages.tsx
@@ -0,0 +1,317 @@
+import { CheckIcon, ClipboardDocumentIcon, CodeBracketSquareIcon } from "@heroicons/react/20/solid";
+import { lazy, Suspense, useState } from "react";
+import { CodeBlock } from "~/components/code/CodeBlock";
+import { Button } from "~/components/primitives/Buttons";
+import { Header3 } from "~/components/primitives/Headers";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import type { DisplayItem, ToolUse } from "./types";
+
+// Lazy load streamdown to avoid SSR issues
+const StreamdownRenderer = lazy(() =>
+ import("streamdown").then((mod) => ({
+ default: ({ children }: { children: string }) => (
+
+ {children}
+
+ ),
+ }))
+);
+
+export function AIChatMessages({ items }: { items: DisplayItem[] }) {
+ return (
+
+ {items.map((item, i) => {
+ switch (item.type) {
+ case "system":
+ return
;
+ case "user":
+ return
;
+ case "tool-use":
+ return
;
+ case "assistant":
+ return
;
+ }
+ })}
+
+ );
+}
+
+// ---------------------------------------------------------------------------
+// Section header (shared across all sections)
+// ---------------------------------------------------------------------------
+
+function SectionHeader({ label, right }: { label: string; right?: React.ReactNode }) {
+ return (
+
+
{label}
+ {right &&
{right}
}
+
+ );
+}
+
+export function ChatBubble({ children }: { children: React.ReactNode }) {
+ return (
+
+ {children}
+
+ );
+}
+
+// ---------------------------------------------------------------------------
+// System
+// ---------------------------------------------------------------------------
+
+function SystemSection({ text }: { text: string }) {
+ const [expanded, setExpanded] = useState(false);
+ const isLong = text.length > 150;
+ const preview = isLong ? text.slice(0, 150) + "..." : text;
+ const displayText = expanded || !isLong ? text : preview;
+
+ return (
+
+
setExpanded(!expanded)}
+ className="text-[10px] text-text-link hover:underline"
+ >
+ {expanded ? "Collapse" : "Expand"}
+
+ ) : undefined
+ }
+ />
+
+
+ {displayText}}>
+ {displayText}
+
+
+
+
+ );
+}
+
+// ---------------------------------------------------------------------------
+// User
+// ---------------------------------------------------------------------------
+
+function UserSection({ text }: { text: string }) {
+ return (
+
+
+
+
+ {text}}>
+ {text}
+
+
+
+
+ );
+}
+
+// ---------------------------------------------------------------------------
+// Assistant response (with markdown/raw toggle)
+// ---------------------------------------------------------------------------
+
+function isJsonString(value: string): boolean {
+ const trimmed = value.trimStart();
+ if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return false;
+ try {
+ JSON.parse(value);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+export function AssistantResponse({
+ text,
+ headerLabel = "Assistant",
+}: {
+ text: string;
+ headerLabel?: string;
+}) {
+ const isJson = isJsonString(text);
+ const [mode, setMode] = useState<"rendered" | "raw">("rendered");
+ const [copied, setCopied] = useState(false);
+
+ function handleCopy() {
+ navigator.clipboard.writeText(text);
+ setCopied(true);
+ setTimeout(() => setCopied(false), 2000);
+ }
+
+ if (isJson) {
+ return (
+
+
+
+
+ );
+ }
+
+ return (
+
+
+
+
+
+ }
+ />
+ {mode === "rendered" ? (
+
+
+ {text}}>
+ {text}
+
+
+
+ ) : (
+
+ )}
+
+
+ {tool.toolName}
+ {tool.resultSummary && (
+ {tool.resultSummary}
+ )}
+
+
+ {availableTabs.length > 0 && (
+ <>
+
+ {availableTabs.map((tab) => (
+
+ ))}
+
+
+ {activeTab === "input" && hasInput && (
+
+
+
+ )}
+
+ {activeTab === "output" && hasResult && (
+
+
+
+ )}
+
+ {activeTab === "details" && hasDetails && (
+
+ {tool.description && (
+
{tool.description}
+ )}
+ {tool.parametersJson && (
+
+
+ Parameters schema
+
+
+
+ )}
+
+ )}
+
+ )}
+
+
+
+ {aiData.provider !== "unknown" && }
+ {aiData.resolvedProvider && (
+
+ )}
+ {aiData.finishReason && }
+ {aiData.serviceTier && }
+ {aiData.toolChoice && }
+ {aiData.toolCount != null && aiData.toolCount > 0 && (
+
+ )}
+ {aiData.messageCount != null && (
+
+ )}
+ {aiData.telemetryMetadata &&
+ Object.entries(aiData.telemetryMetadata).map(([key, value]) => (
+
+ ))}
+
+
+
Stats
+
+
+
+ {aiData.cachedTokens != null && aiData.cachedTokens > 0 && (
+
+ )}
+ {aiData.cacheCreationTokens != null && aiData.cacheCreationTokens > 0 && (
+
+ )}
+ {aiData.reasoningTokens != null && aiData.reasoningTokens > 0 && (
+
+ )}
+
+
+ {aiData.totalCost != null && (
+
+ )}
+ {aiData.msToFirstChunk != null && (
+
+ )}
+ {aiData.tokensPerSecond != null && (
+
+ )}
+
+
+ {label}
+
+ {value}
+ {unit && {unit}}
+
+
+ );
+}
+
+function formatTtfc(ms: number): string {
+ if (ms >= 10_000) {
+ return `${(ms / 1000).toFixed(1)}s`;
+ }
+ return `${Math.round(ms)}ms`;
+}
diff --git a/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx
new file mode 100644
index 00000000000..0070c7ffe3c
--- /dev/null
+++ b/apps/webapp/app/components/runs/v3/ai/AISpanDetails.tsx
@@ -0,0 +1,225 @@
+import { CheckIcon, ClipboardDocumentIcon } from "@heroicons/react/20/solid";
+import { useState } from "react";
+import { Button } from "~/components/primitives/Buttons";
+import { Header3 } from "~/components/primitives/Headers";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { TabButton, TabContainer } from "~/components/primitives/Tabs";
+import { useHasAdminAccess } from "~/hooks/useUser";
+import { CodeBlock } from "~/components/code/CodeBlock";
+import { AIChatMessages, AssistantResponse, ChatBubble } from "./AIChatMessages";
+import { AIStatsSummary, AITagsRow } from "./AIModelSummary";
+import { AIToolsInventory } from "./AIToolsInventory";
+import type { AISpanData, DisplayItem } from "./types";
+
+type AITab = "overview" | "messages" | "tools";
+
+export function AISpanDetails({
+ aiData,
+ rawProperties,
+}: {
+ aiData: AISpanData;
+ rawProperties?: string;
+}) {
+ const [tab, setTab] = useState("overview");
+ const isAdmin = useHasAdminAccess();
+ const toolCount = aiData.toolCount ?? aiData.toolDefinitions?.length ?? 0;
+
+ return (
+
+ {/* Tab bar */}
+
+
+ setTab("overview")}
+ shortcut={{ key: "o" }}
+ >
+ Overview
+
+ setTab("messages")}
+ shortcut={{ key: "m" }}
+ >
+ Messages
+
+ setTab("tools")}
+ shortcut={{ key: "t" }}
+ >
+
+ Tools
+ {toolCount > 0 && (
+
+ {toolCount}
+
+ )}
+
+
+
+
+
+ {/* Tab content */}
+
+ {tab === "overview" && }
+ {tab === "messages" && }
+ {tab === "tools" && }
+
+
+ {/* Footer: Copy raw (admin only) */}
+ {isAdmin && rawProperties &&
}
+
+
+
+
+ {userText && (
+
+
Input
+
+ {userText}
+
+
+ )}
+
+ {outputText &&
}
+ {!outputText && outputObject && (
+
+ Output
+
+
+ )}
+ {outputToolNames.length > 0 && !outputText && !outputObject && (
+
+
Output
+
+
+ Called {outputToolNames.length === 1 ? "tool" : "tools"}:{" "}
+ {outputToolNames.join(", ")}
+
+
+
+ )}
+
+
+ {aiData.items && aiData.items.length > 0 &&
}
+ {showFallbackText &&
}
+ {showFallbackObject && (
+
+ Assistant
+
+
+ )}
+
+
+
+
+ );
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function extractInputOutput(aiData: AISpanData): {
+ userText: string | undefined;
+ outputText: string | undefined;
+ outputObject: string | undefined;
+ outputToolNames: string[];
+} {
+ let userText: string | undefined;
+ let outputText: string | undefined;
+ const outputToolNames: string[] = [];
+
+ if (aiData.items) {
+ for (let i = aiData.items.length - 1; i >= 0; i--) {
+ if (aiData.items[i].type === "user") {
+ userText = (aiData.items[i] as { type: "user"; text: string }).text;
+ break;
+ }
+ }
+
+ for (let i = aiData.items.length - 1; i >= 0; i--) {
+ const item = aiData.items[i];
+ if (item.type === "assistant") {
+ outputText = item.text;
+ break;
+ }
+ if (item.type === "tool-use") {
+ for (const tool of item.tools) {
+ outputToolNames.push(tool.toolName);
+ }
+ break;
+ }
+ }
+ }
+
+ if (!outputText && aiData.responseText) {
+ outputText = aiData.responseText;
+ }
+
+ return { userText, outputText, outputObject: aiData.responseObject, outputToolNames };
+}
+
+function hasAssistantItem(items: DisplayItem[] | undefined): boolean {
+ if (!items) return false;
+ return items.some((item) => item.type === "assistant");
+}
diff --git a/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx b/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx
new file mode 100644
index 00000000000..a329698dd5e
--- /dev/null
+++ b/apps/webapp/app/components/runs/v3/ai/AIToolsInventory.tsx
@@ -0,0 +1,84 @@
+import { useState } from "react";
+import { CodeBlock } from "~/components/code/CodeBlock";
+import type { AISpanData, ToolDefinition } from "./types";
+import { Paragraph } from "~/components/primitives/Paragraph";
+
+export function AIToolsInventory({ aiData }: { aiData: AISpanData }) {
+ const defs = aiData.toolDefinitions ?? [];
+ const calledNames = getCalledToolNames(aiData);
+
+ if (defs.length === 0) {
+ return (
+
+
No tool definitions available for this span.
+
+ {defs.map((def) => {
+ const wasCalled = calledNames.has(def.name);
+ return ;
+ })}
+
+ );
+}
+
+function ToolDefRow({ def, wasCalled }: { def: ToolDefinition; wasCalled: boolean }) {
+ const [showSchema, setShowSchema] = useState(false);
+
+ return (
+
+
+
+
{def.name}
+
{wasCalled ? "called" : "not called"}
+
+
+ {def.description && (
+
{def.description}
+ )}
+
+ {def.parametersJson && (
+
+
+ {showSchema && (
+
+
+
+ )}
+
+ )}
+
-
-
+ {activeFilters.includes("tasks") && (
+
+ )}
+ {activeFilters.includes("queues") &&
}
+ {activeFilters.includes("models") && (
+
+ )}
0 ? tasks : undefined}
queues={queues.length > 0 ? queues : undefined}
+ responseModels={models.length > 0 ? models : undefined}
config={widget.display}
organizationId={organization.id}
projectId={project.id}
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx
index 4d027223f14..f0c3c1d616f 100644
--- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx
+++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.query/ExamplesContent.tsx
@@ -118,6 +118,96 @@ LIMIT 100`,
scope: "environment",
table: "metrics",
},
+ {
+ title: "LLM cost by model (past 7d)",
+ description: "Total cost, input tokens, and output tokens grouped by model over the last 7 days.",
+ query: `SELECT
+ response_model,
+ SUM(total_cost) AS total_cost,
+ SUM(input_tokens) AS input_tokens,
+ SUM(output_tokens) AS output_tokens
+FROM llm_metrics
+WHERE start_time > now() - INTERVAL 7 DAY
+GROUP BY response_model
+ORDER BY total_cost DESC`,
+ scope: "environment",
+ table: "llm_metrics",
+ },
+ {
+ title: "LLM cost over time",
+ description: "Total LLM cost bucketed over time. The bucket size adjusts automatically.",
+ query: `SELECT
+ timeBucket(),
+ SUM(total_cost) AS total_cost
+FROM llm_metrics
+GROUP BY timeBucket
+ORDER BY timeBucket
+LIMIT 1000`,
+ scope: "environment",
+ table: "llm_metrics",
+ },
+ {
+ title: "Most expensive runs by LLM cost (top 50)",
+ description: "Top 50 runs by total LLM cost with token breakdown.",
+ query: `SELECT
+ run_id,
+ task_identifier,
+ SUM(total_cost) AS llm_cost,
+ SUM(input_tokens) AS input_tokens,
+ SUM(output_tokens) AS output_tokens
+FROM llm_metrics
+GROUP BY run_id, task_identifier
+ORDER BY llm_cost DESC
+LIMIT 50`,
+ scope: "environment",
+ table: "llm_metrics",
+ },
+ {
+ title: "LLM calls by provider",
+ description: "Count and cost of LLM calls grouped by AI provider.",
+ query: `SELECT
+ gen_ai_system,
+ count() AS call_count,
+ SUM(total_cost) AS total_cost
+FROM llm_metrics
+GROUP BY gen_ai_system
+ORDER BY total_cost DESC`,
+ scope: "environment",
+ table: "llm_metrics",
+ },
+ {
+ title: "LLM cost by user",
+ description:
+ "Total LLM cost per user from run tags or AI SDK telemetry metadata. Uses metadata.userId which comes from experimental_telemetry metadata or run tags like user:123.",
+ query: `SELECT
+ metadata.userId AS user_id,
+ SUM(total_cost) AS total_cost,
+ SUM(total_tokens) AS total_tokens,
+ count() AS call_count
+FROM llm_metrics
+WHERE metadata.userId != ''
+GROUP BY metadata.userId
+ORDER BY total_cost DESC
+LIMIT 50`,
+ scope: "environment",
+ table: "llm_metrics",
+ },
+ {
+ title: "LLM cost by metadata key",
+ description:
+ "Browse all metadata keys and their LLM cost. Metadata comes from run tags (key:value) and AI SDK telemetry metadata.",
+ query: `SELECT
+ metadata,
+ response_model,
+ total_cost,
+ total_tokens,
+ run_id
+FROM llm_metrics
+ORDER BY start_time DESC
+LIMIT 20`,
+ scope: "environment",
+ table: "llm_metrics",
+ },
];
const tableOptions = querySchemas.map((s) => ({ label: s.name, value: s.name }));
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx
index e02d29b95b5..118226906e1 100644
--- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx
+++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx
@@ -119,7 +119,7 @@ const resizableSettings = {
},
inspector: {
id: "inspector",
- default: "430px" as const,
+ default: "500px" as const,
min: "50px" as const,
},
},
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx
index 29d9e246e2d..ee69419e1b7 100644
--- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx
+++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx
@@ -120,6 +120,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
return typedjson({ ...result, regions: regionsResult.regions });
} catch (error) {
+ logger.error("Failed to load test page", {
+ taskParam,
+ error: error instanceof Error ? error.message : error,
+ stack: error instanceof Error ? error.stack : undefined,
+ });
+
return redirectWithErrorMessage(
v3TestPath({ slug: organizationSlug }, { slug: projectParam }, environment),
request,
diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts
new file mode 100644
index 00000000000..4e8357c886c
--- /dev/null
+++ b/apps/webapp/app/routes/admin.api.v1.llm-models.$modelId.ts
@@ -0,0 +1,152 @@
+import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+
+async function requireAdmin(request: Request) {
+ const authResult = await authenticateApiRequestWithPersonalAccessToken(request);
+ if (!authResult) {
+ throw json({ error: "Invalid or Missing API key" }, { status: 401 });
+ }
+
+ const user = await prisma.user.findUnique({ where: { id: authResult.userId } });
+ if (!user?.admin) {
+ throw json({ error: "You must be an admin to perform this action" }, { status: 403 });
+ }
+
+ return user;
+}
+
+export async function loader({ request, params }: LoaderFunctionArgs) {
+ await requireAdmin(request);
+
+ const model = await prisma.llmModel.findUnique({
+ where: { id: params.modelId },
+ include: {
+ pricingTiers: {
+ include: { prices: true },
+ orderBy: { priority: "asc" },
+ },
+ },
+ });
+
+ if (!model) {
+ return json({ error: "Model not found" }, { status: 404 });
+ }
+
+ return json({ model });
+}
+
+const UpdateModelSchema = z.object({
+ modelName: z.string().min(1).optional(),
+ matchPattern: z.string().min(1).optional(),
+ startDate: z.string().nullable().optional(),
+ pricingTiers: z
+ .array(
+ z.object({
+ name: z.string().min(1),
+ isDefault: z.boolean().default(true),
+ priority: z.number().int().default(0),
+ conditions: z
+ .array(
+ z.object({
+ usageDetailPattern: z.string(),
+ operator: z.enum(["gt", "gte", "lt", "lte", "eq", "neq"]),
+ value: z.number(),
+ })
+ )
+ .default([]),
+ prices: z.record(z.string(), z.number()),
+ })
+ )
+ .optional(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+ await requireAdmin(request);
+
+ const modelId = params.modelId!;
+
+ if (request.method === "DELETE") {
+ const existing = await prisma.llmModel.findUnique({ where: { id: modelId } });
+ if (!existing) {
+ return json({ error: "Model not found" }, { status: 404 });
+ }
+
+ await prisma.llmModel.delete({ where: { id: modelId } });
+ return json({ success: true });
+ }
+
+ if (request.method !== "PUT") {
+ return json({ error: "Method not allowed" }, { status: 405 });
+ }
+
+ let body: unknown;
+ try {
+ body = await request.json();
+ } catch {
+ return json({ error: "Invalid JSON body" }, { status: 400 });
+ }
+
+ const parsed = UpdateModelSchema.safeParse(body);
+
+ if (!parsed.success) {
+ return json({ error: "Invalid request body", details: parsed.error.issues }, { status: 400 });
+ }
+
+ const { modelName, matchPattern, startDate, pricingTiers } = parsed.data;
+
+ // Validate regex if provided — strip (?i) POSIX flag since our registry handles it
+ if (matchPattern) {
+ try {
+ const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern;
+ new RegExp(testPattern);
+ } catch {
+ return json({ error: "Invalid regex in matchPattern" }, { status: 400 });
+ }
+ }
+
+ // Update model + tiers atomically
+ const updated = await prisma.$transaction(async (tx) => {
+ await tx.llmModel.update({
+ where: { id: modelId },
+ data: {
+ ...(modelName !== undefined && { modelName }),
+ ...(matchPattern !== undefined && { matchPattern }),
+ ...(startDate !== undefined && { startDate: startDate ? new Date(startDate) : null }),
+ },
+ });
+
+ if (pricingTiers) {
+ await tx.llmPricingTier.deleteMany({ where: { modelId } });
+
+ for (const tier of pricingTiers) {
+ await tx.llmPricingTier.create({
+ data: {
+ modelId,
+ name: tier.name,
+ isDefault: tier.isDefault,
+ priority: tier.priority,
+ conditions: tier.conditions,
+ prices: {
+ create: Object.entries(tier.prices).map(([usageType, price]) => ({
+ modelId,
+ usageType,
+ price,
+ })),
+ },
+ },
+ });
+ }
+ }
+
+ return tx.llmModel.findUnique({
+ where: { id: modelId },
+ include: {
+ pricingTiers: { include: { prices: true }, orderBy: { priority: "asc" } },
+ },
+ });
+ });
+
+ return json({ model: updated });
+}
diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts
new file mode 100644
index 00000000000..5ca7077e1cc
--- /dev/null
+++ b/apps/webapp/app/routes/admin.api.v1.llm-models.missing.ts
@@ -0,0 +1,33 @@
+import { type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { getMissingLlmModels } from "~/services/admin/missingLlmModels.server";
+
+async function requireAdmin(request: Request) {
+ const authResult = await authenticateApiRequestWithPersonalAccessToken(request);
+ if (!authResult) {
+ throw json({ error: "Invalid or Missing API key" }, { status: 401 });
+ }
+
+ const user = await prisma.user.findUnique({ where: { id: authResult.userId } });
+ if (!user?.admin) {
+ throw json({ error: "You must be an admin to perform this action" }, { status: 403 });
+ }
+
+ return user;
+}
+
+export async function loader({ request }: LoaderFunctionArgs) {
+ await requireAdmin(request);
+
+ const url = new URL(request.url);
+ const lookbackHours = parseInt(url.searchParams.get("lookbackHours") ?? "24", 10);
+
+ if (isNaN(lookbackHours) || lookbackHours < 1 || lookbackHours > 720) {
+ return json({ error: "lookbackHours must be between 1 and 720" }, { status: 400 });
+ }
+
+ const models = await getMissingLlmModels({ lookbackHours });
+
+ return json({ models, lookbackHours });
+}
diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.reload.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.reload.ts
new file mode 100644
index 00000000000..747722b352a
--- /dev/null
+++ b/apps/webapp/app/routes/admin.api.v1.llm-models.reload.ts
@@ -0,0 +1,24 @@
+import { type ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server";
+
+export async function action({ request }: ActionFunctionArgs) {
+ const authResult = await authenticateApiRequestWithPersonalAccessToken(request);
+ if (!authResult) {
+ return json({ error: "Invalid or Missing API key" }, { status: 401 });
+ }
+
+ const user = await prisma.user.findUnique({ where: { id: authResult.userId } });
+ if (!user?.admin) {
+ return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+ }
+
+ if (!llmPricingRegistry) {
+ return json({ error: "LLM cost tracking is disabled" }, { status: 400 });
+ }
+
+ await llmPricingRegistry.reload();
+
+ return json({ success: true, message: "LLM pricing registry reloaded" });
+}
diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.seed.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.seed.ts
new file mode 100644
index 00000000000..805f97ad233
--- /dev/null
+++ b/apps/webapp/app/routes/admin.api.v1.llm-models.seed.ts
@@ -0,0 +1,30 @@
+import { type ActionFunctionArgs, json } from "@remix-run/server-runtime";
+import { seedLlmPricing } from "@internal/llm-pricing";
+import { prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server";
+
+export async function action({ request }: ActionFunctionArgs) {
+ const authResult = await authenticateApiRequestWithPersonalAccessToken(request);
+ if (!authResult) {
+ return json({ error: "Invalid or Missing API key" }, { status: 401 });
+ }
+
+ const user = await prisma.user.findUnique({ where: { id: authResult.userId } });
+ if (!user?.admin) {
+ return json({ error: "You must be an admin to perform this action" }, { status: 403 });
+ }
+
+ const result = await seedLlmPricing(prisma);
+
+ // Reload the in-memory registry after seeding (if enabled)
+ if (llmPricingRegistry) {
+ await llmPricingRegistry.reload();
+ }
+
+ return json({
+ success: true,
+ ...result,
+ message: `Seeded ${result.modelsCreated} models, skipped ${result.modelsSkipped} existing`,
+ });
+}
diff --git a/apps/webapp/app/routes/admin.api.v1.llm-models.ts b/apps/webapp/app/routes/admin.api.v1.llm-models.ts
new file mode 100644
index 00000000000..6305869c605
--- /dev/null
+++ b/apps/webapp/app/routes/admin.api.v1.llm-models.ts
@@ -0,0 +1,141 @@
+import { type ActionFunctionArgs, type LoaderFunctionArgs, json } from "@remix-run/server-runtime";
+import { z } from "zod";
+import { prisma } from "~/db.server";
+import { authenticateApiRequestWithPersonalAccessToken } from "~/services/personalAccessToken.server";
+import { generateFriendlyId } from "~/v3/friendlyIdentifiers";
+
+async function requireAdmin(request: Request) {
+ const authResult = await authenticateApiRequestWithPersonalAccessToken(request);
+ if (!authResult) {
+ throw json({ error: "Invalid or Missing API key" }, { status: 401 });
+ }
+
+ const user = await prisma.user.findUnique({ where: { id: authResult.userId } });
+ if (!user?.admin) {
+ throw json({ error: "You must be an admin to perform this action" }, { status: 403 });
+ }
+
+ return user;
+}
+
+export async function loader({ request }: LoaderFunctionArgs) {
+ await requireAdmin(request);
+
+ const url = new URL(request.url);
+ const page = parseInt(url.searchParams.get("page") ?? "1");
+ const pageSize = parseInt(url.searchParams.get("pageSize") ?? "50");
+
+ const [models, total] = await Promise.all([
+ prisma.llmModel.findMany({
+ where: { projectId: null },
+ include: {
+ pricingTiers: {
+ include: { prices: true },
+ orderBy: { priority: "asc" },
+ },
+ },
+ orderBy: { modelName: "asc" },
+ skip: (page - 1) * pageSize,
+ take: pageSize,
+ }),
+ prisma.llmModel.count({ where: { projectId: null } }),
+ ]);
+
+ return json({ models, total, page, pageSize });
+}
+
+const CreateModelSchema = z.object({
+ modelName: z.string().min(1),
+ matchPattern: z.string().min(1),
+ startDate: z.string().optional(),
+ source: z.enum(["default", "admin"]).optional().default("admin"),
+ pricingTiers: z.array(
+ z.object({
+ name: z.string().min(1),
+ isDefault: z.boolean().default(true),
+ priority: z.number().int().default(0),
+ conditions: z
+ .array(
+ z.object({
+ usageDetailPattern: z.string(),
+ operator: z.enum(["gt", "gte", "lt", "lte", "eq", "neq"]),
+ value: z.number(),
+ })
+ )
+ .default([]),
+ prices: z.record(z.string(), z.number()),
+ })
+ ),
+});
+
+export async function action({ request }: ActionFunctionArgs) {
+ await requireAdmin(request);
+
+ if (request.method !== "POST") {
+ return json({ error: "Method not allowed" }, { status: 405 });
+ }
+
+ let body: unknown;
+ try {
+ body = await request.json();
+ } catch {
+ return json({ error: "Invalid JSON body" }, { status: 400 });
+ }
+
+ const parsed = CreateModelSchema.safeParse(body);
+
+ if (!parsed.success) {
+ return json({ error: "Invalid request body", details: parsed.error.issues }, { status: 400 });
+ }
+
+ const { modelName, matchPattern, startDate, source, pricingTiers } = parsed.data;
+
+ // Validate regex pattern — strip (?i) POSIX flag since our registry handles it
+ try {
+ const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern;
+ new RegExp(testPattern);
+ } catch {
+ return json({ error: "Invalid regex in matchPattern" }, { status: 400 });
+ }
+
+ // Create model + tiers atomically
+ const created = await prisma.$transaction(async (tx) => {
+ const model = await tx.llmModel.create({
+ data: {
+ friendlyId: generateFriendlyId("llm_model"),
+ modelName,
+ matchPattern,
+ startDate: startDate ? new Date(startDate) : null,
+ source,
+ },
+ });
+
+ for (const tier of pricingTiers) {
+ await tx.llmPricingTier.create({
+ data: {
+ modelId: model.id,
+ name: tier.name,
+ isDefault: tier.isDefault,
+ priority: tier.priority,
+ conditions: tier.conditions,
+ prices: {
+ create: Object.entries(tier.prices).map(([usageType, price]) => ({
+ modelId: model.id,
+ usageType,
+ price,
+ })),
+ },
+ },
+ });
+ }
+
+ return tx.llmModel.findUnique({
+ where: { id: model.id },
+ include: {
+ pricingTiers: { include: { prices: true } },
+ },
+ });
+ });
+
+ return json({ model: created }, { status: 201 });
+}
diff --git a/apps/webapp/app/routes/admin.llm-models.$modelId.tsx b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx
new file mode 100644
index 00000000000..e37491a1b4f
--- /dev/null
+++ b/apps/webapp/app/routes/admin.llm-models.$modelId.tsx
@@ -0,0 +1,461 @@
+import { Form, useActionData, useNavigate } from "@remix-run/react";
+import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime";
+import { redirect } from "@remix-run/server-runtime";
+import { typedjson, useTypedLoaderData } from "remix-typedjson";
+import { z } from "zod";
+import { useState } from "react";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
+import { Input } from "~/components/primitives/Input";
+import { Paragraph } from "~/components/primitives/Paragraph";
+import { prisma } from "~/db.server";
+import { requireUserId } from "~/services/session.server";
+import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server";
+
+export const loader = async ({ request, params }: LoaderFunctionArgs) => {
+ const userId = await requireUserId(request);
+ const user = await prisma.user.findUnique({ where: { id: userId } });
+ if (!user?.admin) throw redirect("/");
+
+ const model = await prisma.llmModel.findUnique({
+ where: { friendlyId: params.modelId },
+ include: {
+ pricingTiers: { include: { prices: true }, orderBy: { priority: "asc" } },
+ },
+ });
+
+ if (!model) throw new Response("Model not found", { status: 404 });
+
+ // Convert Prisma Decimal to plain numbers for serialization
+ const serialized = {
+ ...model,
+ pricingTiers: model.pricingTiers.map((t) => ({
+ ...t,
+ prices: t.prices.map((p) => ({ ...p, price: Number(p.price) })),
+ })),
+ };
+
+ return typedjson({ model: serialized });
+};
+
+const SaveSchema = z.object({
+ modelName: z.string().min(1),
+ matchPattern: z.string().min(1),
+ pricingTiersJson: z.string(),
+});
+
+export async function action({ request, params }: ActionFunctionArgs) {
+ const userId = await requireUserId(request);
+ const user = await prisma.user.findUnique({ where: { id: userId } });
+ if (!user?.admin) throw redirect("/");
+
+ const friendlyId = params.modelId!;
+ const existing = await prisma.llmModel.findUnique({ where: { friendlyId } });
+ if (!existing) throw new Response("Model not found", { status: 404 });
+ const modelId = existing.id;
+
+ const formData = await request.formData();
+ const _action = formData.get("_action");
+
+ if (_action === "delete") {
+ await prisma.llmModel.delete({ where: { id: modelId } });
+ await llmPricingRegistry?.reload();
+ return redirect("/admin/llm-models");
+ }
+
+ if (_action === "save") {
+ const raw = Object.fromEntries(formData);
+ const parsed = SaveSchema.safeParse(raw);
+
+ if (!parsed.success) {
+ return typedjson({ error: "Invalid form data", details: parsed.error.issues }, { status: 400 });
+ }
+
+ const { modelName, matchPattern, pricingTiersJson } = parsed.data;
+
+ // Validate regex — strip (?i) POSIX flag since our registry handles it
+ try {
+ const testPattern = matchPattern.startsWith("(?i)") ? matchPattern.slice(4) : matchPattern;
+ new RegExp(testPattern);
+ } catch {
+ return typedjson({ error: "Invalid regex in matchPattern" }, { status: 400 });
+ }
+
+ // Parse tiers
+ let pricingTiers: Array<{
+ name: string;
+ isDefault: boolean;
+ priority: number;
+ conditions: Array<{ usageDetailPattern: string; operator: string; value: number }>;
+ prices: Record;
+ }>;
+ try {
+ pricingTiers = JSON.parse(pricingTiersJson) as typeof pricingTiers;
+ } catch {
+ return typedjson({ error: "Invalid pricing tiers JSON" }, { status: 400 });
+ }
+
+ // Update model
+ await prisma.llmModel.update({
+ where: { id: modelId },
+ data: { modelName, matchPattern },
+ });
+
+ // Replace tiers
+ await prisma.llmPricingTier.deleteMany({ where: { modelId } });
+ for (const tier of pricingTiers) {
+ await prisma.llmPricingTier.create({
+ data: {
+ modelId,
+ name: tier.name,
+ isDefault: tier.isDefault,
+ priority: tier.priority,
+ conditions: tier.conditions,
+ prices: {
+ create: Object.entries(tier.prices).map(([usageType, price]) => ({
+ modelId,
+ usageType,
+ price,
+ })),
+ },
+ },
+ });
+ }
+
+ await llmPricingRegistry?.reload();
+ return typedjson({ success: true });
+ }
+
+ return typedjson({ error: "Unknown action" }, { status: 400 });
+}
+
+export default function AdminLlmModelDetailRoute() {
+ const { model } = useTypedLoaderData();
+ const actionData = useActionData<{ success?: boolean; error?: string; details?: unknown[] }>();
+ const navigate = useNavigate();
+
+ const [modelName, setModelName] = useState(model.modelName);
+ const [matchPattern, setMatchPattern] = useState(model.matchPattern);
+ const [testInput, setTestInput] = useState("");
+ const [tiers, setTiers] = useState(() =>
+ model.pricingTiers.map((t) => ({
+ name: t.name,
+ isDefault: t.isDefault,
+ priority: t.priority,
+ conditions: (t.conditions ?? []) as Array<{
+ usageDetailPattern: string;
+ operator: string;
+ value: number;
+ }>,
+ prices: Object.fromEntries(t.prices.map((p) => [p.usageType, p.price])),
+ }))
+ );
+
+ // Test regex match
+ let testResult: boolean | null = null;
+ if (testInput) {
+ try {
+ const pattern = matchPattern.startsWith("(?i)")
+ ? matchPattern.slice(4)
+ : matchPattern;
+ testResult = new RegExp(pattern, "i").test(testInput);
+ } catch {
+ testResult = null;
+ }
+ }
+
+ return (
+
+
+
+
{model.modelName}
+
+
+ {model.source ?? "default"}
+
+
+ Back to list
+
+
+
+
+
+
+ {/* Delete section */}
+
+
+
+
+
+
+ {/* Prices */}
+
+
+ Prices (per token)
+
+
+ {Object.entries(tier.prices).map(([usageType, price]) => (
+
+ {usageType}
+ {
+ const val = parseFloat(e.target.value);
+ if (!isNaN(val)) {
+ onChange({
+ ...tier,
+ prices: { ...tier.prices, [usageType]: val },
+ });
+ }
+ }}
+ />
+
+
+ ))}
+
+
+ {/* Add price */}
+
+
+ {newUsageType && (
+
+ )}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Missing models
+
+
+
+ Add model
+
+
+
+
+ {/* Model tester */}
+
+
+
+
+
+
+
+ {testResult !== undefined && testResult !== null && (
+
+
+ Testing: {testResult.modelString}
+
+ {testResult.match ? (
+
+ Match:{" "}
+
+ {testResult.match.modelName}
+
+
+ ) : (
+
+ No match found — this model has no pricing data
+
+ )}
+
+ )}
+
+
+
+
+ {total} global models (page {page} of {pageCount})
+
+
+
+
+
+
+
+ Model Name
+ Source
+ Input $/tok
+ Output $/tok
+ Other prices
+
+
+
+ {models.length === 0 ? (
+
+ No models found
+
+ ) : (
+ models.map((model) => {
+ // Get default tier prices
+ const defaultTier =
+ model.pricingTiers.find((t) => t.isDefault) ?? model.pricingTiers[0];
+ const priceMap = defaultTier
+ ? Object.fromEntries(defaultTier.prices.map((p) => [p.usageType, p.price]))
+ : {};
+ const inputPrice = priceMap["input"];
+ const outputPrice = priceMap["output"];
+ const otherPrices = defaultTier
+ ? defaultTier.prices.filter(
+ (p) => p.usageType !== "input" && p.usageType !== "output"
+ )
+ : [];
+
+ return (
+
+
+
+ {model.modelName}
+
+
+
+
+ {model.source ?? "default"}
+
+
+
+
+ {inputPrice != null ? formatPrice(inputPrice) : "-"}
+
+
+
+
+ {outputPrice != null ? formatPrice(outputPrice) : "-"}
+
+
+
+ {otherPrices.length > 0 ? (
+ p.usageType).join(", ")}>
+ +{otherPrices.length} more
+
+ ) : (
+ -
+ )}
+
+
+ );
+ })
+ )}
+
+
+
+
+
+ {/* Header */}
+
+
+
{modelName}
+
+ Missing pricing — {samples.length} sample span{samples.length !== 1 ? "s" : ""} from
+ last {lookbackHours}h
+
+
+
+
+ Add pricing
+
+
+ Back to missing
+
+
+
+
+ {error && (
+
+ {error}
+
+ )}
+
+ {/* Token types summary */}
+ {tokenSummary.length > 0 && (
+
+
+ Token types seen across samples
+
+
+ {tokenSummary.map((t) => (
+
+ {t.key}
+
+ {t.min === t.max ? t.min.toLocaleString() : `${t.min.toLocaleString()}-${t.max.toLocaleString()}`}
+
+
+ ))}
+
+
+ These are the token usage types that need pricing entries (at minimum: input, output).
+
+
+ )}
+
+ {/* Provider-reported costs */}
+ {providerCosts.length > 0 && (
+
+
+ Provider-reported cost data found in {providerCosts.length} span{providerCosts.length !== 1 ? "s" : ""}
+
+
+ {providerCosts.map((c, i) => (
+
+ {c.source}
+ ${c.cost.toFixed(6)}
+
+ ({c.inputTokens.toLocaleString()} in + {c.outputTokens.toLocaleString()} out)
+
+
+ ))}
+
+ {providerCosts[0]?.estimatedInputPrice != null && (
+
+
+ Estimated per-token rates (assuming ~3x output/input ratio):
+
+
+ input: {providerCosts[0].estimatedInputPrice.toExponential(4)}
+ output: {(providerCosts[0].estimatedOutputPrice ?? 0).toExponential(4)}
+
+
+ Cross-reference with the provider's pricing page before using these estimates.
+
+
+ )}
+
+ )}
+
+ {/* Prompt section */}
+
+
+
+ Claude Code prompt — paste this to have it add pricing for this model
+
+
+
+
+ {prompt}
+
+
+
+ {/* Sample spans */}
+
+
+ Sample spans ({samples.length})
+
+ {samples.map((s) => {
+ const expanded = expandedSpans.has(s.span_id);
+ let parsedAttrs: Record
| null = null;
+ try {
+ parsedAttrs = JSON.parse(s.attributes_text) as Record;
+ } catch {
+ // ignore
+ }
+
+ return (
+
+
+ {expanded && parsedAttrs && (
+
+
+ {JSON.stringify(parsedAttrs, null, 2)}
+
+
+ )}
+
+
+
+
Missing LLM Models
+
+ Back to models
+
+
+
+
+ Models appearing in spans without cost enrichment. These models need pricing data added.
+
+
+ {/* Lookback selector */}
+
+ Lookback:
+ {LOOKBACK_OPTIONS.map((opt) => (
+
+ {opt.label}
+
+ ))}
+
+
+ {error && (
+
+ {error}
+
+ )}
+
+
+ {models.length} unpriced model{models.length !== 1 ? "s" : ""} found in the last{" "}
+ {lookbackHours < 24
+ ? `${lookbackHours}h`
+ : lookbackHours < 168
+ ? `${lookbackHours / 24}d`
+ : `${Math.round(lookbackHours / 24)}d`}
+
+
+
+
+
+ Model Name
+ Provider
+ Span Count
+ Actions
+
+
+
+ {models.length === 0 ? (
+
+ All models have pricing data
+
+ ) : (
+ models.map((m) => (
+
+ ))
+ )}
+
+
+
+
+
New LLM Model
+
+ Back to list
+
+
+
+
+
+
+
+
+
Prices (per token)
+
+ {Object.entries(tier.prices).map(([usageType, price]) => (
+
+ {usageType}
+ {
+ const val = parseFloat(e.target.value);
+ if (!isNaN(val)) {
+ onChange({ ...tier, prices: { ...tier.prices, [usageType]: val } });
+ }
+ }}
+ />
+
+
+ ))}
+
+
+
+
+ {newUsageType && (
+
+ )}
+
+
+
@@ -276,9 +279,13 @@ function SpanBody({
/>
)}
-
+ {isAiGeneration ? (
-
+ ) : (
+
+
+
+ )}
);
}
@@ -1155,6 +1162,35 @@ function RunError({ error }: { error: TaskRunError }) {
}
}
+function CollapsibleProperties({ code }: { code: string }) {
+ const [open, setOpen] = useState(false);
+ return (
+
+
+ {open && (
+
+
+
+ )}
+