diff --git a/package.json b/package.json index 26e582f4..a4cabfce 100644 --- a/package.json +++ b/package.json @@ -18,9 +18,9 @@ "typecheck": "tsc --noEmit", "security": "pnpm audit && audit-ci", "lint": "oxlint --no-error-on-unmatched-pattern", - "lint:fix": "oxlint --no-error-on-unmatched-pattern --fix", - "fmt": "oxfmt --check", - "fmt:fix": "oxfmt --write", + "lint:fix": "oxlint --fix --no-error-on-unmatched-pattern", + "fmt": "oxfmt --check --no-error-on-unmatched-pattern", + "fmt:fix": "oxfmt --write --no-error-on-unmatched-pattern", "test": "pnpm --filter *app --color=always test", "test:e2e": "pnpm --filter *app --color=always test:e2e", "test:e2e:component": "pnpm --filter *app --color=always test:e2e:component", diff --git a/packages/app/src/app/api/v1/feedback/route.ts b/packages/app/src/app/api/v1/feedback/route.ts index f55e8458..58897538 100644 --- a/packages/app/src/app/api/v1/feedback/route.ts +++ b/packages/app/src/app/api/v1/feedback/route.ts @@ -5,6 +5,7 @@ import { utf8ToBytes } from '@noble/ciphers/utils.js'; import { getWriteDb } from '@semianalysisai/inferencex-db/connection'; import { type Cipher, createCipher, loadKey } from '@semianalysisai/inferencex-db/lib/encryption'; +import { trackServer } from '@/lib/analytics-server'; import { parseFeedbackBody } from './parse'; const aadFor = (column: string) => utf8ToBytes(`user_feedback:${column}`); @@ -55,6 +56,7 @@ export async function POST(request: Request) { return new NextResponse(null, { status: 204 }); } + const pagePath = body.pagePath ?? null; let cipher: Cipher; let sql: ReturnType; try { @@ -62,6 +64,12 @@ export async function POST(request: Request) { sql = getWriteDb(); } catch (error) { console.error('feedback: misconfigured', error); + trackServer('feedback_submission_failed', { + error_code: 'E_CRYPTO', + error_name: error instanceof Error ? error.name : 'Unknown', + error_message: (error instanceof Error ? error.message : String(error)).slice(0, 500), + page_path: pagePath, + }); return serverError('E_CRYPTO'); } @@ -94,6 +102,12 @@ export async function POST(request: Request) { `; } catch (error) { console.error('feedback: insert failed', error); + trackServer('feedback_submission_failed', { + error_code: 'E_INSERT', + error_name: error instanceof Error ? error.name : 'Unknown', + error_message: (error instanceof Error ? error.message : String(error)).slice(0, 500), + page_path: pagePath, + }); return serverError('E_INSERT'); } diff --git a/packages/app/src/app/compare-per-dollar/[slug]/page.tsx b/packages/app/src/app/compare-per-dollar/[slug]/page.tsx index deede577..b5feb98c 100644 --- a/packages/app/src/app/compare-per-dollar/[slug]/page.tsx +++ b/packages/app/src/app/compare-per-dollar/[slug]/page.tsx @@ -11,7 +11,6 @@ import { compareModelDisplayLabel, parseCompareSlug, } from '@/lib/compare-slug'; -import { getAllComparableCompareSlugs } from '@/lib/compare-availability'; import { getGpuSpecs } from '@/lib/constants'; import { buildBreadcrumbJsonLd, @@ -36,14 +35,6 @@ interface Props { searchParams: Promise>; } -export async function generateStaticParams() { - // Mirror the /compare route's static params — only (model, pair) combos with - // benchmark data on both sides. Direct URL hits to non-enumerated combos - // still render via the dynamic SSR path (with the empty-state fallback). - const slugs = await getAllComparableCompareSlugs(); - return slugs.map(({ modelSlug, a, b }) => ({ slug: canonicalCompareSlug(modelSlug, a, b) })); -} - export async function generateMetadata({ params }: Props): Promise { const { slug } = await params; const parsed = parseCompareSlug(slug); @@ -84,7 +75,9 @@ export default async function ComparePerDollarPage({ params, searchParams }: Pro // alias model resolution, GPU alphabetical order — but redirect target lives // under /compare-per-dollar/. Query string is preserved across the hop. const canonical = canonicalCompareSlug(parsed.model.slug, parsed.a, parsed.b); - if (canonical !== slug) { + // canonical is always lowercase; compare against lowercased input so mixed-case + // URLs don't emit a fresh 308 + CDN cache entry every hit. + if (canonical !== slug.toLowerCase()) { const qs = Object.entries(sp) .flatMap(([k, v]) => { if (Array.isArray(v)) return v.map((vv) => [k, vv] as const); diff --git a/packages/app/src/app/compare-per-dollar/[slug]/performance-per-dollar.png/route.tsx b/packages/app/src/app/compare-per-dollar/[slug]/performance-per-dollar.png/route.tsx index 775f287c..0022785a 100644 --- a/packages/app/src/app/compare-per-dollar/[slug]/performance-per-dollar.png/route.tsx +++ b/packages/app/src/app/compare-per-dollar/[slug]/performance-per-dollar.png/route.tsx @@ -2,6 +2,7 @@ import { ImageResponse } from 'next/og'; import { HW_REGISTRY } from '@semianalysisai/inferencex-constants'; +import { trackServer } from '@/lib/analytics-server'; import { pickPairDefaults } from '@/lib/compare-pair-defaults'; import { canonicalCompareSlug, parseCompareSlug } from '@/lib/compare-slug'; import { @@ -190,336 +191,363 @@ export async function GET( ); } - return new ImageResponse( -
-
-
+ try { + return new ImageResponse( +
+
+
+
+ InferenceX Performance per Dollar +
+
+ {parsed.model.label} +
+
+ {aLabel} vs {bLabel} | Cost per Million Tokens +
+
- InferenceX Performance per Dollar -
-
- {parsed.model.label} -
-
- {aLabel} vs {bLabel} | Cost per Million Tokens -
-
-
-
- DEFAULT WORKLOAD -
-
- {workload || 'Default comparison'} -
-
- Lower cost is better +
+ DEFAULT WORKLOAD +
+
+ {workload || 'Default comparison'} +
+
+ Lower cost is better +
-
-
-
- +
- - {yAxis.ticks.map((tick) => { - const y = scaleY(tick); - return ( - + + {yAxis.ticks.map((tick) => { + const y = scaleY(tick); + return ( + + ); + })} + {plottedRows.map((row) => { + const x = scaleX(row.target); + return ( + + ); + })} + {renderSeriesPath(aSeries.leftExt, COLORS.a, true)} + {renderSeriesPath(aSeries.rightExt, COLORS.a, true)} + {renderSeriesPath(aSeries.matched, COLORS.a, false)} + {renderSeriesPath(bSeries.leftExt, COLORS.b, true)} + {renderSeriesPath(bSeries.rightExt, COLORS.b, true)} + {renderSeriesPath(bSeries.matched, COLORS.b, false)} + {aHighlightPoints.map((point, index) => ( + - ); - })} - {plottedRows.map((row) => { - const x = scaleX(row.target); - return ( - ( + - ); - })} - {renderSeriesPath(aSeries.leftExt, COLORS.a, true)} - {renderSeriesPath(aSeries.rightExt, COLORS.a, true)} - {renderSeriesPath(aSeries.matched, COLORS.a, false)} - {renderSeriesPath(bSeries.leftExt, COLORS.b, true)} - {renderSeriesPath(bSeries.rightExt, COLORS.b, true)} - {renderSeriesPath(bSeries.matched, COLORS.b, false)} - {aHighlightPoints.map((point, index) => ( - + ))} + + {yAxis.ticks.map((tick) => ( +
+ {moneyForStep(tick, yStep)} +
))} - {bHighlightPoints.map((point, index) => ( - + {plottedRows.map((row) => ( +
+ {row.target} +
))} - - {yAxis.ticks.map((tick) => ( -
- {moneyForStep(tick, yStep)} -
- ))} - {plottedRows.map((row) => ( + {showRangeEndpoints && hasLeftExtension && ( +
+ {Math.round(xMin)} +
+ )} + {showRangeEndpoints && hasRightExtension && ( +
+ {Math.round(xMax)} +
+ )}
- {row.target} + Interactivity (tok/s/user)
- ))} - {showRangeEndpoints && hasLeftExtension && ( -
- {Math.round(xMin)} -
- )} - {showRangeEndpoints && hasRightExtension && ( -
- {Math.round(xMax)} -
- )} + {showRangeEndpoints && ( +
+ Dashed segments extend to each SKU's operating envelope, where cost rises steeply +
+ )} +
+
- Interactivity (tok/s/user) -
- {showRangeEndpoints && ( -
- Dashed segments extend to each SKU's operating envelope, where cost rises steeply +
+ Matched Interactivity
- )} +
+ + + {aLabel} + + + + {bLabel} + +
+ {plottedRows.length > 0 ? ( + plottedRows.map((row) => ( +
+
+ {row.target} tok/s/user +
+
+ + {row.a ? money(row.a.cost) : 'N/A'} + + + {row.b ? money(row.b.cost) : 'N/A'} + +
+
+ )) + ) : ( +
+ No matched cost data available. +
+ )} +
-
- Matched Interactivity -
-
- - - {aLabel} - - - - {bLabel} - -
- {plottedRows.length > 0 ? ( - plottedRows.map((row) => ( -
-
- {row.target} tok/s/user -
-
- - {row.a ? money(row.a.cost) : 'N/A'} - - - {row.b ? money(row.b.cost) : 'N/A'} - -
-
- )) - ) : ( -
- No matched cost data available. -
- )} + + Owning-hyperscaler TCO | interpolated from benchmark results + + + inferencex.semianalysis.com +
-
- -
- - Owning-hyperscaler TCO | interpolated from benchmark results - - - inferencex.semianalysis.com - -
-
, - { - ...SIZE, - headers: { - 'Cache-Control': 'public, s-maxage=3600, stale-while-revalidate=86400', +
, + { + ...SIZE, + headers: { + 'Cache-Control': 'public, s-maxage=3600, stale-while-revalidate=86400', + }, }, - }, - ); + ); + } catch (error) { + // Satori can throw on a font fetch failure, malformed JSX layout, or a + // dataset that produces NaN/Infinity geometry. Capture which slug broke so + // we can find broken categories before a crawler hits them — without this, + // failures only surface as opaque Vercel 500s. + const message = error instanceof Error ? error.message : String(error); + trackServer('compare_per_dollar_png_render_failed', { + slug, + model: parsed.model.slug, + a: parsed.a, + b: parsed.b, + sequence, + precision, + error_name: error instanceof Error ? error.name : 'Unknown', + error_message: message.slice(0, 500), + }); + // 502 (not 500): the route itself is reachable, the downstream renderer + // failed. Short cache so a retry within the hour pulls a fixed render + // instead of pinning the failure. + return new Response('PNG render failed', { + status: 502, + headers: { 'Cache-Control': 'public, s-maxage=60' }, + }); + } } diff --git a/packages/app/src/app/compare/[slug]/page.tsx b/packages/app/src/app/compare/[slug]/page.tsx index d6c660ec..b957028e 100644 --- a/packages/app/src/app/compare/[slug]/page.tsx +++ b/packages/app/src/app/compare/[slug]/page.tsx @@ -11,7 +11,6 @@ import { compareModelDisplayLabel, parseCompareSlug, } from '@/lib/compare-slug'; -import { getAllComparableCompareSlugs } from '@/lib/compare-availability'; import { buildBreadcrumbJsonLd, buildJsonLd, @@ -35,14 +34,6 @@ interface Props { searchParams: Promise>; } -export async function generateStaticParams() { - // Only enumerate (model, pair) combos with benchmark data on both sides. - // Direct URL hits to non-enumerated combos still render via the dynamic - // SSR path (with the empty-state fallback). - const slugs = await getAllComparableCompareSlugs(); - return slugs.map(({ modelSlug, a, b }) => ({ slug: canonicalCompareSlug(modelSlug, a, b) })); -} - export async function generateMetadata({ params }: Props): Promise { const { slug } = await params; const parsed = parseCompareSlug(slug); @@ -88,7 +79,10 @@ export default async function ComparePage({ params, searchParams }: Props) { // redirect — the original PR #351 redirect dropped these, but with bare slugs // now redirecting unconditionally we need to keep them. const canonical = canonicalCompareSlug(parsed.model.slug, parsed.a, parsed.b); - if (canonical !== slug) { + // canonical is always lowercase; compare against lowercased input so mixed-case + // URLs (e.g. /compare/H100-vs-H200) don't emit a fresh 308 + CDN cache entry + // every hit when they actually match the canonical content. + if (canonical !== slug.toLowerCase()) { const qs = Object.entries(sp) .flatMap(([k, v]) => { if (Array.isArray(v)) return v.map((vv) => [k, vv] as const); diff --git a/packages/app/src/components/feedback-viewer/FeedbackViewer.tsx b/packages/app/src/components/feedback-viewer/FeedbackViewer.tsx index 5e9c309f..0969e9d8 100644 --- a/packages/app/src/components/feedback-viewer/FeedbackViewer.tsx +++ b/packages/app/src/components/feedback-viewer/FeedbackViewer.tsx @@ -16,12 +16,11 @@ import { import { useFeedbackList } from '@/hooks/api/use-feedback-list'; import type { FeedbackListRow } from '@/lib/api'; import { track } from '@/lib/analytics'; +import { relockFeatureGate } from '@/lib/use-feature-gate'; import { Button } from '@/components/ui/button'; import { Card } from '@/components/ui/card'; import { Input } from '@/components/ui/input'; -const FEATURE_GATE_KEY = 'inferencex-feature-gate'; - interface DecryptedRow { id: string; createdAt: string; @@ -132,8 +131,7 @@ export default function FeedbackViewer() { size="sm" className="h-7 gap-1.5 text-xs text-muted-foreground" onClick={() => { - localStorage.removeItem(FEATURE_GATE_KEY); - window.dispatchEvent(new Event('inferencex:feature-gate:locked')); + relockFeatureGate(); track('feedback_viewer_relocked'); router.push('/inference'); }} diff --git a/packages/app/src/components/gpu-power/GpuPowerDisplay.tsx b/packages/app/src/components/gpu-power/GpuPowerDisplay.tsx index dbc1144b..31e2538b 100644 --- a/packages/app/src/components/gpu-power/GpuPowerDisplay.tsx +++ b/packages/app/src/components/gpu-power/GpuPowerDisplay.tsx @@ -22,6 +22,8 @@ import { SelectValue, } from '@/components/ui/select'; +import { relockFeatureGate } from '@/lib/use-feature-gate'; + import GpuCorrelationChart from './GpuCorrelationChart'; import GpuMetricsChart from './GpuPowerChart'; import GpuStatsTable from './GpuStatsTable'; @@ -35,7 +37,6 @@ import { } from './types'; const GPU_COLORS = d3.schemeTableau10; -const FEATURE_GATE_KEY = 'inferencex-feature-gate'; type GpuMetricsView = 'chart' | 'correlation'; @@ -248,8 +249,7 @@ export default function GpuMetricsDisplay() { size="sm" className="h-7 gap-1.5 text-xs text-muted-foreground" onClick={() => { - localStorage.removeItem(FEATURE_GATE_KEY); - window.dispatchEvent(new Event('inferencex:feature-gate:locked')); + relockFeatureGate(); track('powerx_relocked'); router.push('/inference'); }} diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index beed5e0a..13b22951 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -306,7 +306,12 @@ export function useChartData( .map((d: InferenceData) => { const yValue = (d[metricKey] as { y: number })?.y ?? d.y; const roof = (d[metricKey] as { roof: boolean })?.roof ?? false; - const xValue = (d as any)[xAxisField] ?? d.x; + // xAxisField is `keyof AggDataEntry`; InferenceData embeds those + // fields via `Partial>`, so a typed + // accessor catches a future field rename (silent fallthrough to + // d.x would otherwise mask the regression). + const xCandidate = (d as Partial)[xAxisField]; + const xValue = typeof xCandidate === 'number' ? xCandidate : d.x; return { ...d, x: xValue, diff --git a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts index 2be3e380..a0e17a1d 100644 --- a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts +++ b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.test.ts @@ -161,11 +161,11 @@ describe('interpolateMetricAtInteractivity', () => { makePoint({ x: 20, tpPerGpu: { y: 800, roof: false } }), makePoint({ x: 40, tpPerGpu: { y: 600, roof: false } }), ]; - // jOutput is not set on these points — extractMetric returns null, so metric values are 0 + // jOutput is not set on these points. Returning 0 would render a flat + // zero-line that looks like real data (the bug F4 fixed); return null + // so the trend chart can show a gap instead. const result = interpolateMetricAtInteractivity(points, 30, 'jOutput'); - // With all metric values being 0, the result should be 0 (clamped from spline of zeros) - expect(result).not.toBeNull(); - expect(result!).toBe(0); + expect(result).toBeNull(); }); it('handles two frontier points at close x values', () => { diff --git a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts index 6158935a..e35dcd4b 100644 --- a/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts +++ b/packages/app/src/components/inference/hooks/useInterpolatedTrendData.ts @@ -15,7 +15,10 @@ import { rowToAggDataEntry } from '@/lib/benchmark-transform'; import type { BenchmarkRow } from '@/lib/api'; import type { Model, Sequence } from '@/lib/data-mappings'; -const wrapMetric = (n: number): { y: number } => ({ y: n }); +// Trend points never sit on a roofline — they're synthetic per-(date, config) +// aggregates, not the per-load Pareto-frontier points the chart marks. Hardcode +// roof:false so the field shape lines up with InferenceData without a cast. +const wrapMetric = (n: number): { y: number; roof: boolean } => ({ y: n, roof: false }); /** * Build a lightweight InferenceData-compatible point from a raw BenchmarkRow. @@ -38,8 +41,11 @@ function rowToLightweightPoint(row: BenchmarkRow): InferenceData | null { const outTokPerHr = (outputTput * 3600) / 1_000_000; const inTokPerHr = (inputTput * 3600) / 1_000_000; - // Build metric objects matching InferenceData shape - return { + // Build metric objects matching InferenceData shape. Measured-power keys are + // only set when the runner-side aggregate_power.py emitted them — leaving the + // field undefined lets extractMetric return null and the trend show a real + // gap instead of a flat-zero line. + const point: InferenceData = { x: m.median_intvty ?? 0, y: tput, hwKey, @@ -65,7 +71,17 @@ function rowToLightweightPoint(row: BenchmarkRow): InferenceData | null { jTotal: wrapMetric(power > 0 && tput ? (power * 1000) / tput : 0), ...(outputTput ? { jOutput: wrapMetric(power > 0 ? (power * 1000) / outputTput : 0) } : {}), ...(inputTput ? { jInput: wrapMetric(power > 0 ? (power * 1000) / inputTput : 0) } : {}), - } as unknown as InferenceData; + ...(typeof entry.avg_power_w === 'number' + ? { measuredAvgPower: { y: entry.avg_power_w, roof: false } } + : {}), + ...(typeof entry.joules_per_output_token === 'number' + ? { measuredJPerOutputToken: { y: entry.joules_per_output_token, roof: false } } + : {}), + ...(typeof entry.joules_per_total_token === 'number' + ? { measuredJPerTotalToken: { y: entry.joules_per_total_token, roof: false } } + : {}), + }; + return point; } /** @@ -104,9 +120,17 @@ export function interpolateMetricAtInteractivity( : null; } - // Extract metric values from frontier points + // Extract metric values from frontier points. If ANY point is missing the + // metric (e.g. measured-power keys on a row that predates aggregate_power.py), + // bail out — silently coercing nulls to zero would render a flat-zero trend + // line that looks like real data. const xs = sorted.map((p) => p.x); - const metricYs = sorted.map((p) => extractMetric(p, metricKey) ?? 0); + const metricYs: number[] = []; + for (const p of sorted) { + const v = extractMetric(p, metricKey); + if (v === null) return null; + metricYs.push(v); + } // Monotone cubic Hermite spline interpolation const slopes = monotoneSlopes(xs, metricYs); diff --git a/packages/app/src/components/inference/hooks/useTrendData.ts b/packages/app/src/components/inference/hooks/useTrendData.ts index 868af23b..5e89014f 100644 --- a/packages/app/src/components/inference/hooks/useTrendData.ts +++ b/packages/app/src/components/inference/hooks/useTrendData.ts @@ -3,6 +3,7 @@ import { useEffect, useMemo, useRef, useState } from 'react'; import { sequenceToIslOsl } from '@semianalysisai/inferencex-constants'; import type { + AggDataEntry, InferenceData, TrackedConfig, TrendDataPoint, @@ -110,9 +111,16 @@ export function useTrendData( const metricObj = point[metricKey]; if (!metricObj || typeof metricObj !== 'object' || !('y' in metricObj)) continue; - // Use the dynamic x-axis field if provided (e.g. TTFT instead of E2EL) + // Use the dynamic x-axis field if provided (e.g. TTFT instead of E2EL). + // Typed accessor: a future AggDataEntry field rename would silently fall + // through to point.x without this — narrow on `number` so non-scalar + // metric structs (roofline {y, roof}) can't sneak into the x-axis value. const xField = xAxisFieldByChartType?.[chartType]; - const xValue = xField ? ((point as any)[xField] ?? point.x) : point.x; + let xValue = point.x; + if (xField) { + const xCandidate = (point as Partial)[xField as keyof AggDataEntry]; + if (typeof xCandidate === 'number') xValue = xCandidate; + } if (!accumulator.has(config.id)) accumulator.set(config.id, new Map()); accumulator.get(config.id)!.set(date, { diff --git a/packages/app/src/components/json-ld.test.ts b/packages/app/src/components/json-ld.test.ts index 0cc9c819..34feaa4b 100644 --- a/packages/app/src/components/json-ld.test.ts +++ b/packages/app/src/components/json-ld.test.ts @@ -47,6 +47,24 @@ describe('JsonLd', () => { expect(JSON.parse(body)).toEqual(data); }); + it('escapes > so an HTML parser cannot mistake a literal > for the end of a comment/CDATA', () => { + const data = { note: 'a > b' }; + const html = render(data); + const body = scriptBody(html); + expect(body).toContain(String.raw`\u003e`); + expect(body).not.toContain('>'); + expect(JSON.parse(body)).toEqual(data); + }); + + it('escapes & so the payload cannot smuggle entity references through the parser', () => { + const data = { note: 'Tom & Jerry' }; + const html = render(data); + const body = scriptBody(html); + expect(body).toContain(String.raw`\u0026`); + expect(body).not.toContain('&'); + expect(JSON.parse(body)).toEqual(data); + }); + it('does NOT HTML-escape quotes (Google would reject " in JSON-LD)', () => { const html = render({ name: 'GB200' }); const body = scriptBody(html); diff --git a/packages/app/src/components/json-ld.tsx b/packages/app/src/components/json-ld.tsx index dc02e327..10b4e055 100644 --- a/packages/app/src/components/json-ld.tsx +++ b/packages/app/src/components/json-ld.tsx @@ -1,6 +1,7 @@ // Inline JSON-LD ` -// breakout if any string field ever contains one. +// children would HTML-escape the payload). Escapes `<`, `>`, and `&` per the +// HTML5 spec for `` breakout and +// keeps the payload valid if any string field ever contains one of those. export function JsonLd({ data }: { data: object }) { const json = JSON.stringify(data); if (!json) return null; @@ -8,7 +9,10 @@ export function JsonLd({ data }: { data: object }) {